diff --git a/.gitattributes b/.gitattributes index 1c5d7d9706fac3c57713818e4dfaf962e212867a..aab059b82dd65fece68d95c08c83fca43383e44c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -152,3 +152,50 @@ qo464509/episode_rosbags/episode_4_2025-01-01-10-55-58.bag filter=lfs diff=lfs m 3o6eq09y/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text ix23zko1/episode_rosbags/episode_4_2024-12-31-23-27-07.bag filter=lfs diff=lfs merge=lfs -text ix23zko1/episode_rosbags/episode_0_2024-12-31-23-23-23.bag filter=lfs diff=lfs merge=lfs -text +ix23zko1/episode_rosbags/episode_3_2024-12-31-23-25-55.bag filter=lfs diff=lfs merge=lfs -text +ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/episode_rosbags/episode_1_2024-12-31-23-24-18.bag filter=lfs diff=lfs merge=lfs -text +ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ix23zko1/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9rj28b8z/episode_rosbags/episode_4_2024-12-31-22-24-34.bag filter=lfs diff=lfs merge=lfs -text +9rj28b8z/episode_rosbags/episode_0_2024-12-31-22-21-47.bag filter=lfs diff=lfs merge=lfs -text +9rj28b8z/episode_rosbags/episode_3_2024-12-31-22-23-49.bag filter=lfs diff=lfs merge=lfs -text +9rj28b8z/episode_rosbags/episode_1_2024-12-31-22-22-27.bag filter=lfs diff=lfs merge=lfs -text +9rj28b8z/episode_rosbags/episode_2_2024-12-31-22-23-09.bag filter=lfs diff=lfs merge=lfs -text +ix23zko1/episode_rosbags/episode_2_2024-12-31-23-25-04.bag filter=lfs diff=lfs merge=lfs -text +gwnrzhmx/episode_rosbags/episode_0_2024-12-31-22-15-30.bag filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +gwnrzhmx/episode_rosbags/episode_1_2024-12-31-22-16-10.bag filter=lfs diff=lfs merge=lfs -text +gwnrzhmx/episode_rosbags/episode_4_2024-12-31-22-18-19.bag filter=lfs diff=lfs merge=lfs -text +gwnrzhmx/episode_rosbags/episode_3_2024-12-31-22-17-38.bag filter=lfs diff=lfs merge=lfs -text +gwnrzhmx/episode_rosbags/episode_2_2024-12-31-22-16-50.bag filter=lfs diff=lfs merge=lfs -text +wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 filter=lfs diff=lfs merge=lfs -text +nkdxms1x/episode_rosbags/episode_1_2024-12-31-21-04-39.bag filter=lfs diff=lfs merge=lfs -text +wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +wj867hap/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3o6eq09y/episode_rosbags/episode_4_2024-12-31-21-17-59.bag filter=lfs diff=lfs merge=lfs -text +nkdxms1x/episode_rosbags/episode_0_2024-12-31-21-03-48.bag filter=lfs diff=lfs merge=lfs -text diff --git a/3o6eq09y/episode_rosbags/episode_4_2024-12-31-21-17-59.bag b/3o6eq09y/episode_rosbags/episode_4_2024-12-31-21-17-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..6c3d83490f2d84e9d05247561896704b72191a30 --- /dev/null +++ b/3o6eq09y/episode_rosbags/episode_4_2024-12-31-21-17-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28c39c0ec81c6900dfdbbf95137fac781179dc0730e6c4ca42409212fa865039 +size 1409306555 diff --git a/9rj28b8z/episode_rosbags/episode_0_2024-12-31-22-21-47.bag b/9rj28b8z/episode_rosbags/episode_0_2024-12-31-22-21-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..5294ef0f1cee0aa882c5002dddd1aee51c006a20 --- /dev/null +++ b/9rj28b8z/episode_rosbags/episode_0_2024-12-31-22-21-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5cf3782d366b6f6f9db0701e2d3a27a0cc320efa9addb4efb0baae4bc186d1 +size 1409355580 diff --git a/9rj28b8z/episode_rosbags/episode_1_2024-12-31-22-22-27.bag b/9rj28b8z/episode_rosbags/episode_1_2024-12-31-22-22-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..a0487026c5d502c0e4a17254d59a18b910ecc595 --- /dev/null +++ b/9rj28b8z/episode_rosbags/episode_1_2024-12-31-22-22-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd701941b2f3d4e7e453717995c7a6191670ac4b34e4d358272162f0b1dc4ba2 +size 1414217113 diff --git a/9rj28b8z/episode_rosbags/episode_2_2024-12-31-22-23-09.bag b/9rj28b8z/episode_rosbags/episode_2_2024-12-31-22-23-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..3f95d1bb8645974e6675ed60ed3ec088effd1765 --- /dev/null +++ b/9rj28b8z/episode_rosbags/episode_2_2024-12-31-22-23-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f1419f4c80b861332a83e2d3f18a6bee21c3c56b07b50d6896efe00092190c +size 1410517765 diff --git a/9rj28b8z/episode_rosbags/episode_3_2024-12-31-22-23-49.bag b/9rj28b8z/episode_rosbags/episode_3_2024-12-31-22-23-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..d697d6a401b3d1ff051dce0677796366cfe0d438 --- /dev/null +++ b/9rj28b8z/episode_rosbags/episode_3_2024-12-31-22-23-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:217be4351cb9abe19f2c0157cd141e536607d68eda02589138435d2e11400643 +size 1414366392 diff --git a/9rj28b8z/episode_rosbags/episode_4_2024-12-31-22-24-34.bag b/9rj28b8z/episode_rosbags/episode_4_2024-12-31-22-24-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..15fe0e894caf248fcf88f4248a3ae2aa8bc003fe --- /dev/null +++ b/9rj28b8z/episode_rosbags/episode_4_2024-12-31-22-24-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc9813f6e1e3e6d3f9ad7c63e10e6667201f473d40e7d21881b429449665c9b +size 1410653010 diff --git a/gwnrzhmx/episode_rosbags/episode_0_2024-12-31-22-15-30.bag b/gwnrzhmx/episode_rosbags/episode_0_2024-12-31-22-15-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..e116f78360c68699fd7c4ed466246b3e96d56a90 --- /dev/null +++ b/gwnrzhmx/episode_rosbags/episode_0_2024-12-31-22-15-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:274fa9b38db8a38fab17a2e000d6a1ab398ee3862beea707eab227d9159b27ea +size 1408280363 diff --git a/gwnrzhmx/episode_rosbags/episode_1_2024-12-31-22-16-10.bag b/gwnrzhmx/episode_rosbags/episode_1_2024-12-31-22-16-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..363e6592a60f9d2b933d46c4821afe3ed374d4a1 --- /dev/null +++ b/gwnrzhmx/episode_rosbags/episode_1_2024-12-31-22-16-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9258078f0ba4b0a5164dc594109e09d34c80af47f11f899c5138200c8a7f764d +size 1414773977 diff --git a/gwnrzhmx/episode_rosbags/episode_2_2024-12-31-22-16-50.bag b/gwnrzhmx/episode_rosbags/episode_2_2024-12-31-22-16-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..38136c5dd6a5d7ca5a7b6662fa5276b617a878cb --- /dev/null +++ b/gwnrzhmx/episode_rosbags/episode_2_2024-12-31-22-16-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ff081aeae7ea717ae592a106de73354f1a8e766aa8a43984ec27de308ab3b7 +size 1443058978 diff --git a/gwnrzhmx/episode_rosbags/episode_3_2024-12-31-22-17-38.bag b/gwnrzhmx/episode_rosbags/episode_3_2024-12-31-22-17-38.bag new file mode 100644 index 0000000000000000000000000000000000000000..69d5e7632f53d904a5d17fe2bdb477d6f03d0cd9 --- /dev/null +++ b/gwnrzhmx/episode_rosbags/episode_3_2024-12-31-22-17-38.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3d0cf20b4016a8c03eb0fdba29880b35b8b219aa7deb4eb31a1884025dae4e +size 1409266391 diff --git a/gwnrzhmx/episode_rosbags/episode_4_2024-12-31-22-18-19.bag b/gwnrzhmx/episode_rosbags/episode_4_2024-12-31-22-18-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..b3120d5b4b9f8d72b34cd8d07e0938b937557052 --- /dev/null +++ b/gwnrzhmx/episode_rosbags/episode_4_2024-12-31-22-18-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80036b6280e4b5ddb2fd8de71fb7504994f780163ff042705abb85b34174ffec +size 1410837938 diff --git a/ix23zko1/.hydra/config.yaml b/ix23zko1/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/ix23zko1/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ix23zko1/.hydra/hydra.yaml b/ix23zko1/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c629cc1fe540a08230621abb647d51629c8a9923 --- /dev/null +++ b/ix23zko1/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ix23zko1/.hydra/overrides.yaml b/ix23zko1/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ix23zko1/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ix23zko1/episode_rosbags/aligned_depth_to_color_K.npy b/ix23zko1/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ix23zko1/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ix23zko1/episode_rosbags/color_K.npy b/ix23zko1/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ix23zko1/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ix23zko1/episode_rosbags/episode_1_2024-12-31-23-24-18.bag b/ix23zko1/episode_rosbags/episode_1_2024-12-31-23-24-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..b42a2340debaeb9186ec635f6826c2623a13e57f --- /dev/null +++ b/ix23zko1/episode_rosbags/episode_1_2024-12-31-23-24-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:239769abf0705817af1f286ece339a024ca86ae3ca3e91e4dbab8909b163c01c +size 1417661515 diff --git a/ix23zko1/episode_rosbags/episode_2_2024-12-31-23-25-04.bag b/ix23zko1/episode_rosbags/episode_2_2024-12-31-23-25-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..aba532e68520eec085e9e74b88d0a9d40518441b --- /dev/null +++ b/ix23zko1/episode_rosbags/episode_2_2024-12-31-23-25-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f240bf3c13ea3eb2fc56d7ef6f434502c4d740dcfbdad142d90404e7b9a26d5e +size 1416761782 diff --git a/ix23zko1/episode_rosbags/episode_3_2024-12-31-23-25-55.bag b/ix23zko1/episode_rosbags/episode_3_2024-12-31-23-25-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..4a51c088488295b317dded471ed57706e01a4565 --- /dev/null +++ b/ix23zko1/episode_rosbags/episode_3_2024-12-31-23-25-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189fc7f964dc4545ddafe1e596c2897316dac656589aef0848c1b34f42ce886a +size 1416613606 diff --git a/ix23zko1/eval_video/0_eval.mp4 b/ix23zko1/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad837fcd542a79ce3f39df4356bafd9c1ec05c34 --- /dev/null +++ b/ix23zko1/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeec50fdf1118de79305b75dbc730bef96e223c42b63cc7fd6c2e0932e19edfb +size 1167823 diff --git a/ix23zko1/eval_video/1_eval.mp4 b/ix23zko1/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9f108551a566a24ec16bb476922f9ea13a6c0e4 --- /dev/null +++ b/ix23zko1/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00cd606e45f670b2df9cb4720e6edcc75ceb2f32ba3051bbdee3558c24dd2db8 +size 1101377 diff --git a/ix23zko1/eval_video/2_eval.mp4 b/ix23zko1/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18fc986ae60fbf69c95ccdd7a353351e88088189 --- /dev/null +++ b/ix23zko1/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ca0a8506093efa659f9d2fbb0fae3841fc3573aaf49314cfd07fcb920ab7666 +size 1154695 diff --git a/ix23zko1/eval_video/3_eval.mp4 b/ix23zko1/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9181f5c951cdf535e09e66434ef6952e3ad8de7a --- /dev/null +++ b/ix23zko1/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b8025834b9f25779448e4060e96f4d6772b8a78044b25a3a1a4533f5b4211cd +size 1254048 diff --git a/ix23zko1/eval_video/4_eval.mp4 b/ix23zko1/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1359afa2fe69f32a611665ce60b28b2b6fe4d08e --- /dev/null +++ b/ix23zko1/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e48861ffe574038dd2b64475e47765707e4011acf7de964a2bb8265cf5950acc +size 779902 diff --git a/ix23zko1/tb/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 b/ix23zko1/tb/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 new file mode 100644 index 0000000000000000000000000000000000000000..dde4706084d07d5e982b31fe0773e73d7493b4cf --- /dev/null +++ b/ix23zko1/tb/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa983872358b68063c82da6dbcf40e29edb89deaf958c94ee88ad083d016742 +size 6539 diff --git a/ix23zko1/wandb/debug-internal.log b/ix23zko1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ca8f2f65c2200cce729a62d89c8b8e80759a40e5 --- /dev/null +++ b/ix23zko1/wandb/debug-internal.log @@ -0,0 +1,824 @@ +2024-12-31 23:22:51,413 INFO StreamThr :1764142 [internal.py:wandb_internal():85] W&B internal server running at pid: 1764142, started at: 2024-12-31 23:22:51.412894 +2024-12-31 23:22:51,415 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:22:51,427 INFO WriterThread:1764142 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb +2024-12-31 23:22:51,428 DEBUG SenderThread:1764142 [sender.py:send():379] send: header +2024-12-31 23:22:51,438 DEBUG SenderThread:1764142 [sender.py:send():379] send: run +2024-12-31 23:22:51,672 INFO SenderThread:1764142 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files +2024-12-31 23:22:51,673 INFO SenderThread:1764142 [sender.py:_start_run_threads():1188] run started: ix23zko1 with start time 1735705371.413732 +2024-12-31 23:22:51,706 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:22:51,707 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:22:51,750 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:22:51,755 DEBUG HandlerThread:1764142 [system_info.py:__init__():26] System info init +2024-12-31 23:22:51,755 DEBUG HandlerThread:1764142 [system_info.py:__init__():41] System info init done +2024-12-31 23:22:51,755 INFO HandlerThread:1764142 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:22:51,755 INFO SystemMonitor:1764142 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:22:51,755 INFO HandlerThread:1764142 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:22:51,756 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:22:51,756 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:22:51,757 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:22:51,757 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:22:51,758 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:22:51,769 DEBUG HandlerThread:1764142 [system_info.py:probe():152] Probing system +2024-12-31 23:22:51,771 DEBUG HandlerThread:1764142 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:probe():200] Probing system done +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:22:51.769978', 'startedAt': '2025-01-01T04:22:51.405701', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.998375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4500.015, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 579.1383628845215}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:22:51,777 INFO HandlerThread:1764142 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:22:51,777 INFO HandlerThread:1764142 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:_save_code():44] Saving code +2024-12-31 23:22:51,779 DEBUG HandlerThread:1764142 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:22:51,779 DEBUG HandlerThread:1764142 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:22:51,841 DEBUG HandlerThread:1764142 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:22:51,842 INFO HandlerThread:1764142 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:22:51,845 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:22:52,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:22:52,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:22:52,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:22:52,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:22:52,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:52,102 INFO HandlerThread:1764142 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/tb" + save: true +} +control { + relay_id: "383dac9a791f486ca65aa2a860b70512" +} +_info { + stream_id: "ix23zko1" +} + +2024-12-31 23:22:52,102 DEBUG HandlerThread:1764142 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:22:52,112 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/a3p5h8pv-wandb-metadata.json +2024-12-31 23:22:52,119 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/7w2ikdem-diff.patch +2024-12-31 23:22:52,173 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/y1cyo7vq-code/FISH/eval_robot.py +2024-12-31 23:22:52,250 DEBUG SenderThread:1764142 [sender.py:send():379] send: telemetry +2024-12-31 23:22:52,250 DEBUG SenderThread:1764142 [sender.py:send():379] send: tbrecord +2024-12-31 23:22:52,251 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:22:52,251 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 with policy live +2024-12-31 23:22:52,473 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/yhwx3t7z-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:22:52,674 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code +2024-12-31 23:22:53,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:54,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:54,674 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:55,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:56,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:57,062 DEBUG SenderThread:1764142 [sender.py:send():379] send: config +2024-12-31 23:22:57,062 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:22:57,109 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:58,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:58,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:59,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:00,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:00,676 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:01,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:02,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:02,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:02,676 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:03,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:04,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:04,677 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:05,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:06,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:06,678 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:07,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:07,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:07,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:07,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:08,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:09,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:10,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:11,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:12,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:13,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:13,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:14,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:15,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:16,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:17,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:18,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:18,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:19,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:20,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:20,682 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:21,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:22,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:22,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:22,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:22,683 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:23,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:23,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:23,684 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:23:24,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:24,684 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:25,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:26,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:27,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:28,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:29,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:29,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:30,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:31,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:32,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:33,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:34,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:34,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:35,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:36,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:37,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:37,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:37,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:38,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:38,693 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:39,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:39,189 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:40,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:41,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:42,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:43,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:44,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:44,193 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:44,695 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:45,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:46,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:46,695 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:47,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:48,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:49,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:50,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:50,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:50,697 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:51,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:51,542 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:23:51,542 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 with policy now +2024-12-31 23:23:51,543 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:23:51,544 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:23:51,544 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:23:51,544 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:23:51,697 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos +2024-12-31 23:23:51,758 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:23:51,759 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:23:51,807 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/cb3bkrvo-media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:52,067 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/muwwa1t9-media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:52,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:52,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:52,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:52,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:53,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:54,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:55,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:55,200 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:56,118 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:56,699 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:57,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:58,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:59,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:00,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:00,700 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:01,010 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:01,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:02,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:03,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:04,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:05,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:06,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:06,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:07,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:07,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:07,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:08,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:09,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:10,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:11,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:11,197 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:12,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:13,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:14,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:14,705 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:24:15,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:15,455 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/nzlocrr2-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:24:16,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:16,537 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:16,706 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:17,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:18,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:19,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:20,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:20,707 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:21,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:21,766 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:24:21,766 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:22,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:22,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:22,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:23,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:24,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:25,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:26,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:27,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:27,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:28,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:29,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:30,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:31,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:32,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:33,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:33,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:34,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:35,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:36,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:36,712 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:37,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:37,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:37,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:38,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:38,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:39,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:40,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:41,128 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:42,128 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:43,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:44,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:44,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:45,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:46,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:46,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:46,824 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:24:46,825 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 with policy now +2024-12-31 23:24:46,828 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:24:46,829 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:24:46,829 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:24:46,830 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:24:47,095 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gauz3vyo-media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:47,130 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:24:47,999 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gnwggf25-media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:48,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:48,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:49,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:49,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:50,131 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:51,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:51,760 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:24:52,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:52,102 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:52,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:52,718 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:53,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:54,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:54,385 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:54,718 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:55,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:56,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:57,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:58,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:59,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:00,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:00,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:01,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:01,530 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:01,530 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:01,532 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:01,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:01,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:02,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:02,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:03,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:04,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:04,722 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:05,132 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:05,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:06,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:06,722 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:07,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:07,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:07,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:08,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:09,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:10,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:10,237 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:11,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:12,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:13,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:14,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:14,725 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:15,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:16,065 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:16,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:17,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:18,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:19,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:20,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:21,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:21,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:21,760 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:25:22,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:22,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:22,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:23,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:24,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:25,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:26,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:26,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:27,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:28,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:29,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:30,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:31,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:32,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:32,146 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:33,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:33,195 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:25:33,195 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 with policy now +2024-12-31 23:25:33,198 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:25:33,199 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:33,199 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:33,199 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:33,452 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gimsickx-media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:33,731 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:33,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:33,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:25:34,009 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/sxvw80v3-media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:34,147 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:34,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:35,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:36,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:37,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:37,103 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:37,151 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:37,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:38,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:38,733 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:39,153 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:40,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:40,734 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:41,151 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:42,153 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:43,110 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:43,155 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:44,154 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:45,155 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:46,157 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:47,157 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:48,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:48,156 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:49,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:50,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:51,161 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:51,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:51,761 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:25:52,000 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/xbzcagni-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:52,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:52,105 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:52,163 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:52,378 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:52,378 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:52,379 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:52,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:52,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:53,161 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:53,497 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:54,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:54,739 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:55,167 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:56,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:56,740 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:57,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:58,162 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:59,107 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:59,162 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:00,163 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:01,176 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:02,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:03,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:04,107 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:04,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:05,175 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:06,175 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:07,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:07,101 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:07,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:08,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:09,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:09,213 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:10,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:11,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:12,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:13,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:14,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:15,109 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:15,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:16,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:17,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:18,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:19,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:20,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:21,108 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:21,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:21,761 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:26:22,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:22,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:22,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:23,200 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:23,746 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:26:23,746 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 with policy now +2024-12-31 23:26:23,749 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:26:23,749 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:26:23,749 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:26:23,750 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:26:24,035 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/cr606cpd-media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:24,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:24,332 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/2nit8asm-media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:24,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:25,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:26,195 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:26,779 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:27,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:28,197 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:28,752 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:29,199 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:30,204 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:31,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:32,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:32,207 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:32,753 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:33,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:34,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:35,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:36,208 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:37,105 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:37,105 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:37,204 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:37,214 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:38,213 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:39,207 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:40,211 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:41,214 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:42,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:43,118 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:43,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:44,219 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:45,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:46,218 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:47,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:48,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:48,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:49,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:50,219 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:51,223 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:51,762 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:26:52,106 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:52,106 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:52,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:53,205 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:53,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:54,230 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:55,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:56,227 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:57,228 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:58,226 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:59,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:59,227 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:00,228 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:01,229 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:02,231 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:03,236 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:03,460 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:03,461 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:03,462 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:03,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:03,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:04,020 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/kssd9jo2-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:04,236 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:04,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:04,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:05,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:06,237 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:06,765 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:07,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:07,102 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:07,238 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:08,239 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:08,766 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:09,234 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:09,903 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:10,234 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:10,766 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:11,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:12,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:13,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:14,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:15,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:15,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:16,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:17,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:18,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:19,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:20,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:20,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:21,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:21,762 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:22,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:22,101 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:22,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:23,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:24,246 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:25,262 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:25,316 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:25,362 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:25,362 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 with policy now +2024-12-31 23:27:25,367 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:25,368 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:25,368 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:25,368 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:25,635 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/xfrfhtq1-media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:25,771 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:25,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:25,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:27:26,031 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/jza0ki16-media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:26,252 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:26,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:27,260 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:28,258 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:29,261 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:30,265 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:30,773 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:31,045 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:31,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:32,266 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:32,774 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:33,264 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:34,267 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:34,775 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:35,274 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:36,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:36,266 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:37,106 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:37,106 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:37,268 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:38,265 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:39,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:40,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:41,191 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:41,279 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:42,276 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:43,268 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:44,276 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:45,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:46,283 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:47,131 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:47,285 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:48,284 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:49,288 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:50,283 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:50,447 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:27:50,447 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:27:50,643 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:50,780 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval +2024-12-31 23:27:50,894 INFO wandb-upload_0:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk7287tw1 +2024-12-31 23:27:50,979 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/len2tr5z-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:51,007 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:27:51,016 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:51,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:51,310 INFO SenderThread:1764142 [sender.py:send_request_log_artifact():1525] logged artifact run-ix23zko1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE5NzgzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODA3Nw==', 'latestArtifact': None}} +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:51,310 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,311 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,311 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:51,311 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,315 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,315 DEBUG SenderThread:1764142 [sender.py:send():379] send: telemetry +2024-12-31 23:27:51,315 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,316 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,316 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,316 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,316 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,317 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,317 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,318 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,318 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,318 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,318 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,319 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,319 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,319 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,319 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,320 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,320 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:27:51,549 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/r0q1x21y-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:51,717 INFO wandb-upload_1:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp28rcpsod +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval +2024-12-31 23:27:52,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:52,113 INFO SenderThread:1764142 [sender.py:send_request_log_artifact():1525] logged artifact run-ix23zko1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE5Nzg0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODA3OQ==', 'latestArtifact': None}} +2024-12-31 23:27:52,113 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:52,113 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json with policy now +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:52,115 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:52,116 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:52,243 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:52,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:52,308 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/r637fazs-media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:52,567 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/s183w4yz-media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:52,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:52,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:53,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:54,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:54,593 DEBUG SenderThread:1764142 [sender.py:send():379] send: exit +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():588] handling runtime: 302 +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():594] send defer +2024-12-31 23:27:54,594 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,594 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:27:54,595 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:27:54,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:27:54,595 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:27:54,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:27:54,595 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:27:54,595 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:27:54,596 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:27:54,596 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:54,651 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,651 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:27:54,652 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,652 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:27:54,783 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:56,784 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,526 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:28:01,526 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:28:01,527 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,527 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:28:01,527 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:28:01,527 DEBUG SenderThread:1764142 [sender.py:send():379] send: summary +2024-12-31 23:28:01,531 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:28:01,531 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,531 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:28:01,532 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:28:01,532 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,532 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:28:01,532 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,532 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:28:01,535 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:28:01,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:01,824 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:28:01,824 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:28:01,825 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:01,825 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,825 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:28:01,825 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,825 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:28:02,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:02,776 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:28:02,776 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:02,776 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:02,776 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:28:02,777 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:02,777 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:28:02,777 INFO SenderThread:1764142 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:28:02,779 INFO SenderThread:1764142 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:28:02,781 INFO SenderThread:1764142 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:28:02,800 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:28:02,800 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:02,800 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:28:02,800 DEBUG SenderThread:1764142 [sender.py:send():379] send: artifact +2024-12-31 23:28:02,825 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:28:02,825 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:03,152 INFO wandb-upload_2:1764142 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp88oy__h_ +2024-12-31 23:28:03,152 INFO wandb-upload_1:1764142 [upload_job.py:push():85] Skipped uploading /tmp/tmpchtkkcxn/wandb-job.json +2024-12-31 23:28:03,250 INFO wandb-upload_0:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaxguhh2y +2024-12-31 23:28:03,596 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE5ODAyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'versionIndex': 398}}} +2024-12-31 23:28:03,805 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:28:03,825 INFO SenderThread:1764142 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml config.yaml +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json wandb-summary.json +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt requirements.txt +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log output.log +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch diff.patch +2024-12-31 23:28:03,828 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:28:03,828 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:28:03,833 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:03,835 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:03,835 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:28:03,836 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:03,836 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:28:03,836 INFO SenderThread:1764142 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:28:04,018 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:28:04,061 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:28:04,065 INFO wandb-upload_3:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt +2024-12-31 23:28:04,071 INFO wandb-upload_4:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:04,075 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/4ixgafw6-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:28:04,275 INFO Thread-11 (_thread_body):1764142 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:28:04,275 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,275 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:28:04,276 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,276 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:28:04,276 INFO SenderThread:1764142 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:28:04,277 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,277 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:28:04,277 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:28:04,502 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,502 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:28:04,502 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send():379] send: final +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send():379] send: footer +2024-12-31 23:28:04,503 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,503 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,503 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:28:04,503 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:04,504 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:04,504 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:28:04,504 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:28:04,505 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:04,506 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:28:04,506 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:28:04,507 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:28:04,560 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:04,560 INFO MainThread:1764142 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:28:04,560 INFO MainThread:1764142 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:28:04,561 INFO MainThread:1764142 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:28:04,561 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:28:04,561 INFO HandlerThread:1764142 [handler.py:finish():882] shutting down handler +2024-12-31 23:28:05,506 INFO WriterThread:1764142 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [sender.py:finish():1615] shutting down sender +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [file_pusher.py:join():175] waiting for file pusher diff --git a/ix23zko1/wandb/debug.log b/ix23zko1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cb1cb8f262e1daf479cdbb8a8beeaae8f33c5920 --- /dev/null +++ b/ix23zko1/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Configure stats pid to 1763844 +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/settings +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug.log +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug-internal.log +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1'} +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():619] starting backend +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():623] setting up manager +2024-12-31 23:22:51,412 INFO MainThread:1763844 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:22:51,413 INFO MainThread:1763844 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:22:51,427 INFO MainThread:1763844 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:22:51,435 INFO MainThread:1763844 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:22:51,706 INFO MainThread:1763844 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:22:51,745 INFO MainThread:1763844 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:22:51,746 INFO MainThread:1763844 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:22:52,101 INFO MainThread:1763844 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:22:52,101 INFO MainThread:1763844 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/tb, True +2024-12-31 23:22:57,061 INFO MainThread:1763844 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:28:05,561 WARNING MsgRouterThr:1763844 [router.py:message_loop():77] message_loop has been closed diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c2f76d86619761267c73c27c37372274b74d11c --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735705371 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..948b8f3435bdf69e7372a20035ba4a55f6e718ca --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch @@ -0,0 +1,86 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..b9e3dfe 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history ++# wandb_run_id: '1002_0' ++ ++# all books, 6/20 demos per book ++# RGBD+mask+act history + wandb_run_id: '1007_0' + ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json new file mode 100644 index 0000000000000000000000000000000000000000..bb86131bad8a6bd968084e2399009a078aa4688e --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[14, 30, 0.4666666666666667, 0.2629978359219579, 0.6768004893902991, 0.3015182316700217, 0.6554286777662317, 0.32652336362493584, 0.6110612675325477, 0.34375845644022573, 0.5951753780055431]]} \ No newline at end of file diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad837fcd542a79ce3f39df4356bafd9c1ec05c34 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeec50fdf1118de79305b75dbc730bef96e223c42b63cc7fd6c2e0932e19edfb +size 1167823 diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9f108551a566a24ec16bb476922f9ea13a6c0e4 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00cd606e45f670b2df9cb4720e6edcc75ceb2f32ba3051bbdee3558c24dd2db8 +size 1101377 diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18fc986ae60fbf69c95ccdd7a353351e88088189 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ca0a8506093efa659f9d2fbb0fae3841fc3573aaf49314cfd07fcb920ab7666 +size 1154695 diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9181f5c951cdf535e09e66434ef6952e3ad8de7a --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b8025834b9f25779448e4060e96f4d6772b8a78044b25a3a1a4533f5b4211cd +size 1254048 diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1359afa2fe69f32a611665ce60b28b2b6fe4d08e --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e48861ffe574038dd2b64475e47765707e4011acf7de964a2bb8265cf5950acc +size 779902 diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..59ee6378f359fee7ed046991bae7ad724e9b9776 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735705377.062350]: resetting environment +[INFO] [1735705377.069575]: cleared current plan +[INFO] [1735705377.071244]: moving to home +[INFO] [1735705378.073267]: reached home +[INFO] [1735705378.074844]: reset action history +[INFO] [1735705380.783974]: environment reset +Starting episode 0 +[INFO] [1735705380.786510]: resetting environment +[INFO] [1735705380.789752]: cleared current plan +[INFO] [1735705380.791144]: moving to home +[INFO] [1735705381.794270]: reached home +[INFO] [1735705381.796008]: reset action history +[INFO] [1735705384.503909]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735705399.733702]: resetting environment +[INFO] [1735705399.737190]: cleared current plan +[INFO] [1735705399.738044]: moving to home +[INFO] [1735705400.740458]: reached home +[INFO] [1735705400.741668]: reset action history +[INFO] [1735705403.450042]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/episode_rosbags/episode_0_2024-12-31-23-23-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1763844.log +started roslaunch server http://158.130.50.37:45197/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1764338] +process[print_text-2]: started with pid [1764362] +process[rosbag_record-3]: started with pid [1764363] +started bagging! +For topic gripper_width: timestamp difference is 102041854 for nearest: 1735705416520001067 - target: 1735705416417959213 at idx 178 +For topic gripper_width: timestamp difference is 106378053 for nearest: 1735705421919963573 - target: 1735705421813585520 at idx 247 +For topic gripper_width: timestamp difference is 140871007 for nearest: 1735705424119961698 - target: 1735705423979090691 at idx 249 +For topic gripper_width: timestamp difference is -192123918 for nearest: 1735705424253291579 - target: 1735705424445415497 at idx 249 +For topic gripper_width: timestamp difference is 111043663 for nearest: 1735705428119965048 - target: 1735705428008921385 at idx 249 +[INFO] [1735705430.503121]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735705431.265008]: Stored episode 1. +[INFO] [1735705431.265242]: Saving video... +[INFO] [1735705431.523601]: Video saved! +Starting episode 1 +[INFO] [1735705431.545691]: resetting environment +[INFO] [1735705431.549961]: cleared current plan +[INFO] [1735705431.550268]: moving to home +[INFO] [1735705435.256597]: reached home +[INFO] [1735705435.257030]: reset action history +[INFO] [1735705437.966370]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735705454.488602]: resetting environment +[INFO] [1735705454.490999]: cleared current plan +[INFO] [1735705454.491157]: moving to home +[INFO] [1735705455.492563]: reached home +[INFO] [1735705455.493627]: reset action history +[INFO] [1735705458.202812]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/episode_rosbags/episode_1_2024-12-31-23-24-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1763844.log +started roslaunch server http://158.130.50.37:37899/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1764475] +process[print_text-5]: started with pid [1764476] +process[rosbag_record-6]: started with pid [1764500] +started bagging! +For topic gripper_width: timestamp difference is -217798505 for nearest: 1735705474253334489 - target: 1735705474471132994 at idx 226 +[INFO] [1735705485.242267]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735705486.048554]: Stored episode 2. +[INFO] [1735705486.049467]: Saving video... +[INFO] [1735705486.788718]: Video saved! +Starting episode 2 +[INFO] [1735705486.831948]: resetting environment +[INFO] [1735705486.847362]: cleared current plan +[INFO] [1735705486.855657]: moving to home +[WARN] [1735705486.574249]: Plan exhausted +[WARN] [1735705486.614203]: Plan exhausted +[WARN] [1735705486.654212]: Plan exhausted +[WARN] [1735705486.694333]: Plan exhausted +[WARN] [1735705486.742095]: Plan exhausted +[WARN] [1735705486.774960]: Plan exhausted +[WARN] [1735705486.814959]: Plan exhausted +[INFO] [1735705490.565387]: reached home +[INFO] [1735705490.568522]: reset action history +[INFO] [1735705493.340520]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735705501.216985]: resetting environment +[INFO] [1735705501.223313]: cleared current plan +[INFO] [1735705501.224358]: moving to home +[INFO] [1735705502.226843]: reached home +[INFO] [1735705502.227553]: reset action history +[INFO] [1735705504.940190]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/episode_rosbags/episode_2_2024-12-31-23-25-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1763844.log +started roslaunch server http://158.130.50.37:45009/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1764612] +process[print_text-8]: started with pid [1764636] +process[rosbag_record-9]: started with pid [1764637] +started bagging! +For topic gripper_width: timestamp difference is 103371654 for nearest: 1735705513019959722 - target: 1735705512916588068 at idx 113 +[INFO] [1735705532.098828]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735705532.830240]: Stored episode 3. +[INFO] [1735705532.836840]: Saving video... +[INFO] [1735705533.151389]: Video saved! +Starting episode 3 +[INFO] [1735705533.205078]: resetting environment +[INFO] [1735705533.226279]: cleared current plan +[INFO] [1735705533.229586]: moving to home +[INFO] [1735705536.946969]: reached home +[INFO] [1735705536.949765]: reset action history +[INFO] [1735705539.797777]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735705551.434893]: resetting environment +[INFO] [1735705551.443621]: cleared current plan +[INFO] [1735705551.445770]: moving to home +[INFO] [1735705552.451352]: reached home +[INFO] [1735705552.455570]: reset action history +[INFO] [1735705555.179552]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/episode_rosbags/episode_3_2024-12-31-23-25-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1763844.log +started roslaunch server http://158.130.50.37:39323/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1764749] +process[print_text-11]: started with pid [1764750] +process[rosbag_record-12]: started with pid [1764774] +started bagging! +[INFO] [1735705582.319831]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1735705583.080423]: Stored episode 4. +[INFO] [1735705583.086799]: Saving video... +[INFO] [1735705583.709671]: Video saved! +Starting episode 4 +[INFO] [1735705583.755128]: resetting environment +[INFO] [1735705583.776053]: cleared current plan +[INFO] [1735705583.778098]: moving to home +[WARN] [1735705583.656499]: Plan exhausted +[WARN] [1735705583.694503]: Plan exhausted +[WARN] [1735705583.736489]: Plan exhausted +[WARN] [1735705583.774212]: Plan exhausted +[INFO] [1735705587.599492]: reached home +[INFO] [1735705587.601034]: reset action history +[INFO] [1735705590.400108]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735705623.207325]: resetting environment +[INFO] [1735705623.232991]: cleared current plan +[INFO] [1735705623.237709]: moving to home +[INFO] [1735705624.243272]: reached home +[INFO] [1735705624.248002]: reset action history +[INFO] [1735705627.029378]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/episode_rosbags/episode_4_2024-12-31-23-27-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1763844.log +started roslaunch server http://158.130.50.37:38139/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1764908] +process[print_text-14]: started with pid [1764909] +process[rosbag_record-15]: started with pid [1764910] +started bagging! +preempting episode +[INFO] [1735705644.150497]: Storing episode... +[print_text-14] killing on exit[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1735705644.999417]: Stored episode 5. +[INFO] [1735705645.002374]: Saving video... +[INFO] [1735705645.312643]: Video saved! +[INFO] [1735705645.372400]: resetting environment +[INFO] [1735705645.396023]: cleared current plan +[INFO] [1735705645.396280]: moving to home +[WARN] [1735705645.335951]: Plan exhausted +[WARN] [1735705645.376273]: Plan exhausted +[INFO] [1735705649.000052]: reached home +[INFO] [1735705649.010500]: reset action history +[INFO] [1735705652.060615]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..607611d7222b02ec7dc347d86f3e38bee6921a22 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:22:51.769978", + "startedAt": "2025-01-01T04:22:51.405701", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4124.998375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.019, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.015, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 579.1383628845215 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5e803e8364d3565a866d75393633caef093029e4 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "eeec50fdf1118de79305b75dbc730bef96e223c42b63cc7fd6c2e0932e19edfb", "size": 1167823, "path": "media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4"}, "global_step": 606, "_timestamp": 1735705670.9173996, "_runtime": 299.5036675930023, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "00cd606e45f670b2df9cb4720e6edcc75ceb2f32ba3051bbdee3558c24dd2db8", "size": 1101377, "path": "media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4"}, "eval/num_success": 0.0, "episode": 30.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "3ca0a8506093efa659f9d2fbb0fae3841fc3573aaf49314cfd07fcb920ab7666", "size": 1154695, "path": "media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4b8025834b9f25779448e4060e96f4d6772b8a78044b25a3a1a4533f5b4211cd", "size": 1254048, "path": "media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e48861ffe574038dd2b64475e47765707e4011acf7de964a2bb8265cf5950acc", "size": 779902, "path": "media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://tl513i0uy8upvergcygdern945umz6808ilf7kwiei2v50acp7lql0kfn9fflelulxvvey7m5tbg29kexez03p4qjk3558emarp6o0qnmst3cerlidqr1ahvk5xfs692/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://mi73bqk23hj6woqibyrjwj87ufuliyx1nnn9oofcz9yio0r89sg46l5m39yssuf1i1kqnvykbm6p6ejffd224d9tm7zk90vi43qart1hgiu7wr5666t7sxvgkjhan6ks:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 14, "total_num_episodes": 30, "total_success_rate": 0.4666666666666667, "total_success_rate_lb_0.01": 0.2629978359219579, "total_success_rate_ub_0.01": 0.6768004893902991, "total_success_rate_lb_0.025": 0.3015182316700217, "total_success_rate_ub_0.025": 0.6554286777662317, "total_success_rate_lb_0.05": 0.32652336362493584, "total_success_rate_ub_0.05": 0.6110612675325477, "total_success_rate_lb_0.1": 0.34375845644022573, "total_success_rate_ub_0.1": 0.5951753780055431, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "cdffd89ce3ca7a2fac895e34b3a4c4f97735368b1ef33cbf174cf8e29f059823", "size": 522, "artifact_path": "wandb-client-artifact://pclj198tnuh6j81i4qzmrtpnx8d8t1hfsnyb4rc6pxcp1xljjnnj6e4a31kymv704oyk9prqr2hqwykd5l06uk5klswfiw89ulrjh8c70l74n9l9a63n2oaaji9r8kzd/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xy9aawejic19yyish95yj95pjimgnhmsxiwh57r37gswzfp6mg3xvo6t0tfhyfsdp20n7bbjndxqrjt8waom2lsj72tj6bhg7qqxujuvnxh0ibsv1gnccqarffowjruw:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 302}, "eval/total_success_rate": 0.46666666865348816, "eval/total_success_rate_lb_0.01": 0.2629978358745575, "eval/total_success_rate_ub_0.01": 0.6768004894256592, "eval/total_success_rate_lb_0.025": 0.3015182316303253, "eval/total_success_rate_ub_0.025": 0.6554286479949951, "eval/total_success_rate_lb_0.05": 0.3265233635902405, "eval/total_success_rate_ub_0.05": 0.6110612750053406, "eval/total_success_rate_lb_0.1": 0.3437584638595581, "eval/total_success_rate_ub_0.1": 0.5951753854751587} \ No newline at end of file diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug-internal.log b/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ca8f2f65c2200cce729a62d89c8b8e80759a40e5 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug-internal.log @@ -0,0 +1,824 @@ +2024-12-31 23:22:51,413 INFO StreamThr :1764142 [internal.py:wandb_internal():85] W&B internal server running at pid: 1764142, started at: 2024-12-31 23:22:51.412894 +2024-12-31 23:22:51,415 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:22:51,427 INFO WriterThread:1764142 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb +2024-12-31 23:22:51,428 DEBUG SenderThread:1764142 [sender.py:send():379] send: header +2024-12-31 23:22:51,438 DEBUG SenderThread:1764142 [sender.py:send():379] send: run +2024-12-31 23:22:51,672 INFO SenderThread:1764142 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files +2024-12-31 23:22:51,673 INFO SenderThread:1764142 [sender.py:_start_run_threads():1188] run started: ix23zko1 with start time 1735705371.413732 +2024-12-31 23:22:51,706 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:22:51,707 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:22:51,750 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:22:51,755 DEBUG HandlerThread:1764142 [system_info.py:__init__():26] System info init +2024-12-31 23:22:51,755 DEBUG HandlerThread:1764142 [system_info.py:__init__():41] System info init done +2024-12-31 23:22:51,755 INFO HandlerThread:1764142 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:22:51,755 INFO SystemMonitor:1764142 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:22:51,755 INFO HandlerThread:1764142 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:22:51,756 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:22:51,756 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:22:51,757 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:22:51,757 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:22:51,758 INFO SystemMonitor:1764142 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:22:51,769 DEBUG HandlerThread:1764142 [system_info.py:probe():152] Probing system +2024-12-31 23:22:51,771 DEBUG HandlerThread:1764142 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:probe():200] Probing system done +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:22:51.769978', 'startedAt': '2025-01-01T04:22:51.405701', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.998375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4500.015, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 579.1383628845215}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:22:51,777 INFO HandlerThread:1764142 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:22:51,777 INFO HandlerThread:1764142 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:22:51,777 DEBUG HandlerThread:1764142 [system_info.py:_save_code():44] Saving code +2024-12-31 23:22:51,779 DEBUG HandlerThread:1764142 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:22:51,779 DEBUG HandlerThread:1764142 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:22:51,841 DEBUG HandlerThread:1764142 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:22:51,842 INFO HandlerThread:1764142 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:22:51,845 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:22:51,845 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:22:52,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:22:52,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:22:52,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:22:52,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:22:52,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:52,102 INFO HandlerThread:1764142 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/tb" + save: true +} +control { + relay_id: "383dac9a791f486ca65aa2a860b70512" +} +_info { + stream_id: "ix23zko1" +} + +2024-12-31 23:22:52,102 DEBUG HandlerThread:1764142 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:22:52,112 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/a3p5h8pv-wandb-metadata.json +2024-12-31 23:22:52,119 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/7w2ikdem-diff.patch +2024-12-31 23:22:52,173 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/y1cyo7vq-code/FISH/eval_robot.py +2024-12-31 23:22:52,250 DEBUG SenderThread:1764142 [sender.py:send():379] send: telemetry +2024-12-31 23:22:52,250 DEBUG SenderThread:1764142 [sender.py:send():379] send: tbrecord +2024-12-31 23:22:52,251 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:22:52,251 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 with policy live +2024-12-31 23:22:52,473 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/yhwx3t7z-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:22:52,674 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH +2024-12-31 23:22:52,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code +2024-12-31 23:22:53,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:54,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:54,674 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:55,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:56,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:57,062 DEBUG SenderThread:1764142 [sender.py:send():379] send: config +2024-12-31 23:22:57,062 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:22:57,109 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:58,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:22:58,675 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:22:59,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:00,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:00,676 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:01,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:02,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:02,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:02,676 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:03,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:04,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:04,677 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:05,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:06,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:06,678 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:07,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:07,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:07,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:07,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:08,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:09,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:10,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:11,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:12,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:13,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:13,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:14,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:15,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:16,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:17,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:18,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:18,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:19,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:20,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:20,682 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:21,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:22,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:22,098 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:22,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:22,683 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:23,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:23,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:23,684 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:23:24,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:24,684 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:25,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:26,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:27,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:28,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:29,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:29,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:30,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:31,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:32,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:33,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:34,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:34,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:35,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:36,112 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:37,098 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:37,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:37,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:38,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:38,693 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:39,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:39,189 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:40,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:41,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:42,113 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:43,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:44,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:44,193 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:44,695 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:45,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:46,114 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:46,695 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:47,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:48,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:49,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:50,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:50,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:50,697 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:51,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:51,542 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:23:51,542 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 with policy now +2024-12-31 23:23:51,543 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:23:51,544 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:23:51,544 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:23:51,544 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:23:51,697 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:23:51,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos +2024-12-31 23:23:51,758 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:23:51,759 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:23:51,807 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/cb3bkrvo-media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:52,067 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/muwwa1t9-media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:23:52,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:23:52,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:23:52,116 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:52,698 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:53,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:54,117 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:55,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:55,200 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:23:56,118 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:56,699 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:23:57,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:58,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:23:59,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:00,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:00,700 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:01,010 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:01,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:02,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:03,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:04,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:05,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:06,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:06,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:07,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:07,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:07,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:08,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:09,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:10,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:11,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:11,197 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:12,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:13,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:14,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:14,705 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:24:15,121 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:15,455 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/nzlocrr2-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:24:16,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:16,537 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:16,706 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:17,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:18,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:19,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:20,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:20,707 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:21,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:21,766 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:24:21,766 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:22,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:22,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:22,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:23,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:24,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:25,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:26,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:27,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:27,123 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:28,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:29,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:30,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:31,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:32,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:33,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:33,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:34,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:35,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:36,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:36,712 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:37,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:37,099 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:37,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:38,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:38,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:39,126 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:40,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:41,128 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:42,128 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:43,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:44,102 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:44,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:45,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:46,129 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:46,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:46,824 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:24:46,825 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 with policy now +2024-12-31 23:24:46,828 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:24:46,829 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:24:46,829 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:24:46,830 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:24:47,095 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gauz3vyo-media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:47,130 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:47,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:24:47,999 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gnwggf25-media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:24:48,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:48,716 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:49,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:49,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:50,131 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:51,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:51,760 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:24:52,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:24:52,102 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:24:52,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:52,718 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:53,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:54,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:54,385 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:24:54,718 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:24:55,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:56,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:57,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:58,133 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:24:59,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:00,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:00,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:01,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:01,530 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:01,530 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:01,532 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:01,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:01,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:02,134 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:02,721 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:03,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:04,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:04,722 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:05,132 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:05,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:06,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:06,722 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:07,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:07,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:07,145 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:08,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:09,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:10,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:10,237 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:11,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:12,135 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:13,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:14,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:14,725 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:15,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:16,065 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:16,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:17,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:18,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:19,136 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:20,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:21,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:21,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:21,760 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:25:22,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:22,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:22,141 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:23,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:24,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:25,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:26,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:26,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:27,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:28,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:29,137 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:30,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:31,138 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:32,099 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:32,146 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:33,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:33,195 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:25:33,195 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 with policy now +2024-12-31 23:25:33,198 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:25:33,199 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:33,199 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:33,199 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:33,452 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/gimsickx-media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:33,731 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:33,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:33,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:25:34,009 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/sxvw80v3-media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:25:34,147 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:34,732 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:35,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:36,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:37,103 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:37,103 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:37,151 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:37,254 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:38,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:38,733 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:39,153 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:40,149 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:40,734 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:41,151 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:42,153 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:43,110 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:43,155 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:44,154 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:45,155 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:46,157 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:47,157 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:48,111 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:48,156 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:49,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:50,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:51,161 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:51,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:51,761 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:25:52,000 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/xbzcagni-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:25:52,104 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:25:52,105 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:25:52,163 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:52,378 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:25:52,378 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:25:52,379 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:25:52,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:25:52,738 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:53,161 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:53,497 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:54,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:54,739 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:55,167 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:56,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:56,740 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:25:57,159 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:58,162 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:25:59,107 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:25:59,162 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:00,163 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:01,176 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:02,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:03,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:04,107 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:04,173 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:05,175 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:06,175 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:07,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:07,101 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:07,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:08,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:09,183 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:09,213 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:10,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:11,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:12,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:13,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:14,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:15,109 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:15,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:16,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:17,184 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:18,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:19,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:20,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:21,108 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:21,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:21,761 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:26:22,100 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:22,100 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:22,185 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:23,200 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:23,746 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:26:23,746 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 with policy now +2024-12-31 23:26:23,749 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:26:23,749 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:26:23,749 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:26:23,750 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:23,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:26:24,035 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/cr606cpd-media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:24,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:24,332 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/2nit8asm-media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:26:24,750 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:25,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:26,195 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:26,779 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:27,194 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:28,197 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:28,752 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:29,199 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:30,204 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:31,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:32,119 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:32,207 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:32,753 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:26:33,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:34,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:35,203 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:36,208 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:37,105 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:37,105 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:37,204 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:37,214 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:38,213 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:39,207 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:40,211 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:41,214 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:42,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:43,118 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:43,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:44,219 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:45,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:46,218 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:47,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:48,125 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:48,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:49,220 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:50,219 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:51,223 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:51,762 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:26:52,106 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:26:52,106 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:26:52,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:53,205 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:53,224 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:54,230 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:55,225 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:56,227 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:57,228 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:58,226 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:26:59,124 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:26:59,227 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:00,228 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:01,229 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:02,231 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:03,236 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:03,460 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:03,461 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:03,462 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:03,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:03,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:04,020 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/kssd9jo2-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:04,236 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:04,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:04,764 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:05,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:06,237 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:06,765 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:07,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:07,102 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:07,238 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:08,239 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:08,766 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:09,234 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:09,903 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:10,234 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:10,766 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:11,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:12,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:13,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:14,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:15,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:15,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:16,235 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:17,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:18,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:19,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:20,120 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:20,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:21,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:21,762 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:22,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:22,101 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:22,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:23,245 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:24,246 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:25,262 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:25,316 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:25,362 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:25,362 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 with policy now +2024-12-31 23:27:25,367 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:25,368 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:25,368 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:25,368 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:25,635 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/xfrfhtq1-media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:25,771 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:25,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:25,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval +2024-12-31 23:27:26,031 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/jza0ki16-media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:27:26,252 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:26,772 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:27,260 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:28,258 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:29,261 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:30,265 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:30,773 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:31,045 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:31,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:32,266 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:32,774 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:33,264 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:34,267 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:34,775 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:35,274 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:36,122 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:36,266 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:37,106 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:37,106 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:37,268 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:38,265 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:39,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:40,269 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:41,191 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:41,279 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:42,276 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:43,268 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:44,276 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:45,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:46,283 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:47,131 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:47,285 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:48,284 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:49,288 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:50,283 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:50,447 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:27:50,447 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:27:50,643 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:50,780 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table +2024-12-31 23:27:50,781 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval +2024-12-31 23:27:50,894 INFO wandb-upload_0:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk7287tw1 +2024-12-31 23:27:50,979 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/len2tr5z-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:51,007 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:27:51,016 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:27:51,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:51,310 INFO SenderThread:1764142 [sender.py:send_request_log_artifact():1525] logged artifact run-ix23zko1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE5NzgzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODA3Nw==', 'latestArtifact': None}} +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:51,310 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:51,310 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,311 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,311 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:51,311 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,315 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,315 DEBUG SenderThread:1764142 [sender.py:send():379] send: telemetry +2024-12-31 23:27:51,315 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,316 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,316 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,316 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,316 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,317 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,317 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,317 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,318 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,318 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,318 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,318 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,319 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,319 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,319 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,319 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,320 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:51,320 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:51,320 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:27:51,549 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/r0q1x21y-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:27:51,717 INFO wandb-upload_1:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp28rcpsod +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:51,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval +2024-12-31 23:27:52,101 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:27:52,113 INFO SenderThread:1764142 [sender.py:send_request_log_artifact():1525] logged artifact run-ix23zko1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE5Nzg0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODA3OQ==', 'latestArtifact': None}} +2024-12-31 23:27:52,113 DEBUG SenderThread:1764142 [sender.py:send():379] send: files +2024-12-31 23:27:52,113 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json with policy now +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:27:52,115 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:52,115 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:52,116 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:27:52,243 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:27:52,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:52,308 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/r637fazs-media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:52,567 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/s183w4yz-media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:27:52,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:52,782 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:27:53,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:54,280 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:27:54,593 DEBUG SenderThread:1764142 [sender.py:send():379] send: exit +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():588] handling runtime: 302 +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:27:54,594 INFO SenderThread:1764142 [sender.py:send_exit():594] send defer +2024-12-31 23:27:54,594 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,594 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:27:54,595 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:27:54,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:27:54,595 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:27:54,595 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:27:54,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:27:54,595 INFO HandlerThread:1764142 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:27:54,595 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:27:54,595 DEBUG SystemMonitor:1764142 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:27:54,596 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:27:54,596 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:27:54,650 INFO HandlerThread:1764142 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send():379] send: stats +2024-12-31 23:27:54,651 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,651 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:27:54,651 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:27:54,651 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:27:54,652 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:27:54,652 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:27:54,783 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:27:56,784 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:28:01,525 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,526 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:28:01,526 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send():379] send: history +2024-12-31 23:28:01,526 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:28:01,527 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,527 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:28:01,527 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:28:01,527 DEBUG SenderThread:1764142 [sender.py:send():379] send: summary +2024-12-31 23:28:01,531 INFO SenderThread:1764142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:28:01,531 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,531 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:28:01,532 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:28:01,532 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,532 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:28:01,532 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,532 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:28:01,535 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:28:01,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:01,824 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:28:01,824 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:28:01,825 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:01,825 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:01,825 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:28:01,825 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:01,825 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:28:02,595 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:02,776 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:28:02,776 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:02,776 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:02,776 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:28:02,777 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:02,777 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:28:02,777 INFO SenderThread:1764142 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:28:02,779 INFO SenderThread:1764142 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:28:02,781 INFO SenderThread:1764142 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:28:02,800 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:28:02,800 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:02,800 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:28:02,800 DEBUG SenderThread:1764142 [sender.py:send():379] send: artifact +2024-12-31 23:28:02,825 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:28:02,825 INFO Thread-12 :1764142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:03,152 INFO wandb-upload_2:1764142 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp88oy__h_ +2024-12-31 23:28:03,152 INFO wandb-upload_1:1764142 [upload_job.py:push():85] Skipped uploading /tmp/tmpchtkkcxn/wandb-job.json +2024-12-31 23:28:03,250 INFO wandb-upload_0:1764142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaxguhh2y +2024-12-31 23:28:03,596 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE5ODAyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'versionIndex': 398}}} +2024-12-31 23:28:03,805 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:28:03,805 INFO SenderThread:1764142 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:28:03,825 INFO SenderThread:1764142 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml config.yaml +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json wandb-summary.json +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt requirements.txt +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log output.log +2024-12-31 23:28:03,826 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/diff.patch diff.patch +2024-12-31 23:28:03,828 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:28:03,828 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 media/videos/eval/4_eval_4_e48861ffe574038dd2b6.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 media/videos/eval/0_eval_0_eeec50fdf1118de79305.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 media/videos/eval/3_eval_3_4b8025834b9f25779448.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 media/videos/eval/2_eval_2_3ca0a8506093efa659f9.mp4 +2024-12-31 23:28:03,829 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 media/videos/eval/1_eval_1_00cd606e45f670b2df9c.mp4 +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json media/table/eval/total_success_rate_ci_6_cdffd89ce3ca7a2fac89.table.json +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:28:03,833 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:28:03,833 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:03,835 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:03,835 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:28:03,836 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:03,836 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:28:03,836 INFO SenderThread:1764142 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:28:04,018 INFO wandb-upload_2:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/config.yaml +2024-12-31 23:28:04,061 INFO wandb-upload_1:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/wandb-summary.json +2024-12-31 23:28:04,065 INFO wandb-upload_3:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/requirements.txt +2024-12-31 23:28:04,071 INFO wandb-upload_4:1764142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/files/output.log +2024-12-31 23:28:04,075 INFO wandb-upload_0:1764142 [upload_job.py:push():130] Uploaded file /tmp/tmpvim0lr5zwandb/4ixgafw6-events.out.tfevents.1735705372.leonmkim-ROG-Strix-G15CS-G15CS.1763844.0 +2024-12-31 23:28:04,275 INFO Thread-11 (_thread_body):1764142 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:28:04,275 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,275 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:28:04,276 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,276 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:28:04,276 INFO SenderThread:1764142 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:28:04,277 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,277 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:28:04,277 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:28:04,277 INFO SenderThread:1764142 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:28:04,502 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,502 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:28:04,502 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:28:04,502 INFO SenderThread:1764142 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send():379] send: final +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send():379] send: footer +2024-12-31 23:28:04,503 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:28:04,503 INFO HandlerThread:1764142 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:28:04,503 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: defer +2024-12-31 23:28:04,503 INFO SenderThread:1764142 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:28:04,503 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:04,504 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:04,504 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:28:04,504 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:28:04,505 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:28:04,506 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:28:04,506 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:28:04,507 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:28:04,560 DEBUG SenderThread:1764142 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:28:04,560 INFO MainThread:1764142 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:28:04,560 INFO MainThread:1764142 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:28:04,561 INFO MainThread:1764142 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:28:04,561 DEBUG HandlerThread:1764142 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:28:04,561 INFO HandlerThread:1764142 [handler.py:finish():882] shutting down handler +2024-12-31 23:28:05,506 INFO WriterThread:1764142 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [sender.py:finish():1615] shutting down sender +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:28:05,560 INFO SenderThread:1764142 [file_pusher.py:join():175] waiting for file pusher diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug.log b/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cb1cb8f262e1daf479cdbb8a8beeaae8f33c5920 --- /dev/null +++ b/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Configure stats pid to 1763844 +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/settings +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug.log +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/wandb/run-20241231_232251-ix23zko1/logs/debug-internal.log +2024-12-31 23:22:51,407 INFO MainThread:1763844 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1'} +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():619] starting backend +2024-12-31 23:22:51,408 INFO MainThread:1763844 [wandb_init.py:init():623] setting up manager +2024-12-31 23:22:51,412 INFO MainThread:1763844 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:22:51,413 INFO MainThread:1763844 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:22:51,427 INFO MainThread:1763844 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:22:51,435 INFO MainThread:1763844 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:22:51,706 INFO MainThread:1763844 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:22:51,745 INFO MainThread:1763844 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:22:51,746 INFO MainThread:1763844 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:22:52,099 INFO MainThread:1763844 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:22:52,101 INFO MainThread:1763844 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:22:52,101 INFO MainThread:1763844 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ix23zko1/tb, True +2024-12-31 23:22:57,061 INFO MainThread:1763844 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:28:05,561 WARNING MsgRouterThr:1763844 [router.py:message_loop():77] message_loop has been closed diff --git a/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb b/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5ee9c8e95aacf9b9776ea6e789bf6bfea796f372 Binary files /dev/null and b/ix23zko1/wandb/run-20241231_232251-ix23zko1/run-ix23zko1.wandb differ diff --git a/ksj6ie6d/episode_rosbags/aligned_depth_to_color_K.npy b/ksj6ie6d/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ksj6ie6d/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ksj6ie6d/episode_rosbags/cam_tf_world.npy b/ksj6ie6d/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ksj6ie6d/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ksj6ie6d/episode_rosbags/color_K.npy b/ksj6ie6d/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ksj6ie6d/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ksj6ie6d/episode_rosbags/depth_K.npy b/ksj6ie6d/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ksj6ie6d/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ksj6ie6d/eval_robot.log b/ksj6ie6d/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0f54686ecc3363a5f58fbbaad552f2922ea5dc50 --- /dev/null +++ b/ksj6ie6d/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 21:08:31,773][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:590: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:08:31,776][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:526: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/ksj6ie6d/eval_video/0_eval.mp4 b/ksj6ie6d/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/ksj6ie6d/eval_video/0_eval.mp4 differ diff --git a/ksj6ie6d/wandb/debug-internal.log b/ksj6ie6d/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3c70aa967d479378de23bdd5eb2165bd508bd9a --- /dev/null +++ b/ksj6ie6d/wandb/debug-internal.log @@ -0,0 +1,664 @@ +2024-12-31 21:08:37,537 INFO StreamThr :1741299 [internal.py:wandb_internal():85] W&B internal server running at pid: 1741299, started at: 2024-12-31 21:08:37.536993 +2024-12-31 21:08:37,538 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:08:37,550 INFO WriterThread:1741299 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb +2024-12-31 21:08:37,551 DEBUG SenderThread:1741299 [sender.py:send():379] send: header +2024-12-31 21:08:37,559 DEBUG SenderThread:1741299 [sender.py:send():379] send: run +2024-12-31 21:08:37,786 INFO SenderThread:1741299 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files +2024-12-31 21:08:37,786 INFO SenderThread:1741299 [sender.py:_start_run_threads():1188] run started: ksj6ie6d with start time 1735697317.537409 +2024-12-31 21:08:37,802 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:08:37,802 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:08:37,875 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:08:37,880 DEBUG HandlerThread:1741299 [system_info.py:__init__():26] System info init +2024-12-31 21:08:37,880 DEBUG HandlerThread:1741299 [system_info.py:__init__():41] System info init done +2024-12-31 21:08:37,880 INFO HandlerThread:1741299 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:08:37,881 INFO SystemMonitor:1741299 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:08:37,881 INFO HandlerThread:1741299 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:08:37,881 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:08:37,882 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:08:37,882 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:08:37,883 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:08:37,883 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:08:37,894 DEBUG HandlerThread:1741299 [system_info.py:probe():152] Probing system +2024-12-31 21:08:37,896 DEBUG HandlerThread:1741299 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:probe():200] Probing system done +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:08:37.895012', 'startedAt': '2025-01-01T02:08:37.529957', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.7145, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4481.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.919, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 500.70812606811523}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:08:37,901 INFO HandlerThread:1741299 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:08:37,901 INFO HandlerThread:1741299 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:_save_code():44] Saving code +2024-12-31 21:08:37,903 DEBUG HandlerThread:1741299 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:08:37,903 DEBUG HandlerThread:1741299 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:08:37,964 DEBUG HandlerThread:1741299 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:08:37,964 INFO HandlerThread:1741299 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:08:37,967 DEBUG SenderThread:1741299 [sender.py:send():379] send: files +2024-12-31 21:08:37,967 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:08:37,967 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:08:37,968 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:08:38,193 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:08:38,193 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:08:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:08:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:38,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:08:38,237 INFO HandlerThread:1741299 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/tb" + save: true +} +control { + relay_id: "7b45fda39ce243519ba995b8b7d508f7" +} +_info { + stream_id: "ksj6ie6d" +} + +2024-12-31 21:08:38,237 DEBUG HandlerThread:1741299 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:08:38,261 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/323z6vdp-diff.patch +2024-12-31 21:08:38,265 INFO wandb-upload_0:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/lk0pmb1b-wandb-metadata.json +2024-12-31 21:08:38,283 INFO wandb-upload_1:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/noqhmg2m-code/FISH/eval_robot.py +2024-12-31 21:08:38,310 DEBUG SenderThread:1741299 [sender.py:send():379] send: telemetry +2024-12-31 21:08:38,310 DEBUG SenderThread:1741299 [sender.py:send():379] send: tbrecord +2024-12-31 21:08:38,311 DEBUG SenderThread:1741299 [sender.py:send():379] send: files +2024-12-31 21:08:38,311 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 with policy live +2024-12-31 21:08:38,502 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/l588opbr-events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH/eval_robot.py +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch +2024-12-31 21:08:38,789 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code +2024-12-31 21:08:38,789 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH +2024-12-31 21:08:39,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:40,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:40,787 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:08:41,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:42,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:43,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:43,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:44,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:45,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:46,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:47,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:48,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:48,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:49,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:50,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:51,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:52,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:08:53,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:08:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:53,264 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:54,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:55,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:56,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:57,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:58,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:59,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:59,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:00,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:01,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:02,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:03,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:04,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:04,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:05,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:06,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:07,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:08,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:08,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:09,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:09,324 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:09,796 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:09:10,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:11,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:12,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:13,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:14,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:15,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:15,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:16,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:17,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:18,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:19,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:20,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:20,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:21,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:22,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:23,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:23,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:24,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:25,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:25,343 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:26,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:27,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:28,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:29,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:30,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:31,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:31,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:32,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:33,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:34,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:35,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:36,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:36,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:37,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:37,884 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:09:37,884 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:09:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:38,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:39,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:40,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:41,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:41,334 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:42,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:43,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:44,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:45,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:46,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:47,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:47,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:48,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:49,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:50,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:51,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:52,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:52,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:53,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:54,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:55,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:56,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:57,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:57,282 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:58,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:59,204 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:00,204 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:01,206 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:02,206 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:03,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:03,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:04,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:05,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:06,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:07,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:07,885 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:10:08,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:08,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:08,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:08,282 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:09,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:10,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:11,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:12,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:13,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:14,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:14,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:15,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:16,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:17,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:18,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:19,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:20,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:20,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:21,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:22,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:23,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:23,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:24,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:24,811 DEBUG SenderThread:1741299 [sender.py:send():379] send: config +2024-12-31 21:10:25,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:25,820 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:26,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:26,818 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:27,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:28,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:29,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:30,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:30,819 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:31,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:31,581 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:32,211 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:33,211 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:34,212 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:34,821 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:35,212 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:36,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:37,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:37,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:37,885 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:10:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:38,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:39,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:40,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:41,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:42,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:42,281 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:43,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:44,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:45,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:46,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:47,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:48,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:48,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:48,825 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:10:49,215 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:50,215 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:51,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:52,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:53,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:53,355 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:54,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:55,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:56,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:57,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:58,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:59,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:59,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:00,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:01,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:02,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:03,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:04,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:04,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:05,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:06,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:07,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:07,886 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:11:08,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:08,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:09,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:09,340 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:10,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:11,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:12,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:13,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:14,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:15,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:15,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:16,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:17,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:18,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:19,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:20,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:20,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:21,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:22,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:23,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:23,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:24,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:25,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:25,272 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:26,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:27,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:28,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:29,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:30,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:31,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:31,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:32,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:33,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:34,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:35,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:36,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:37,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:37,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:37,886 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:11:38,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:38,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:39,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:40,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:41,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:42,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:42,275 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:43,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:44,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:45,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:46,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:47,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:48,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:48,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:49,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:50,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:51,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:52,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:53,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:53,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:53,320 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:54,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:55,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:56,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:57,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:58,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:59,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:59,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:00,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:01,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:02,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:03,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:04,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:04,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:05,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:06,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:07,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:07,887 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:12:08,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:08,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:09,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:09,357 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:10,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:11,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:12,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:13,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:14,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:15,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:15,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:16,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:17,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:18,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:19,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:20,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:20,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:21,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:22,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:23,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:23,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:24,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:25,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:25,277 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:26,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:27,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:28,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:29,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:30,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:31,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:31,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:32,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:33,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:34,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:35,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:36,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:36,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:37,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:37,887 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:12:38,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:38,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:39,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:40,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:41,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:41,308 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:42,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:43,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:44,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:45,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:46,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:47,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:47,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:48,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:49,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:50,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:51,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:52,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:52,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:53,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:53,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:54,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:55,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:56,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:57,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:57,284 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:58,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:59,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:00,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:01,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:02,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:03,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:03,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:04,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:05,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:06,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:07,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:07,888 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:13:08,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:08,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:08,270 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:09,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:10,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:11,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:12,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:13,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:14,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:14,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:15,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:16,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:17,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:18,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:19,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:19,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:20,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:21,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:22,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:23,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:23,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:23,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:24,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:24,265 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:25,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:26,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:27,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:28,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:29,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:30,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:30,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:31,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:32,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:33,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:34,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:35,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:35,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:36,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:37,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:37,888 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:13:38,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:38,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:38,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:39,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:40,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:40,318 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:41,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:42,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:43,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:44,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:45,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:46,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:46,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:47,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:48,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:49,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:50,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:51,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:51,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:52,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:53,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:53,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:53,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:54,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:55,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:56,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:56,268 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:57,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:58,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:59,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:00,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:00,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:01,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:01,368 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:02,074 DEBUG SenderThread:1741299 [sender.py:send():379] send: exit +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:send_exit():586] handling exit code: 1 +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:send_exit():588] handling runtime: 324 +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_exit():594] send defer +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,075 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:14:02,075 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,075 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:14:02,075 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:14:02,076 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:14:02,076 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:14:02,129 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:14:02,130 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,130 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:14:02,130 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:14:02,130 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:14:02,130 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:14:02,131 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,131 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:14:02,131 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:14:02,131 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,131 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:14:02,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:02,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:03,879 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:14:08,276 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,276 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:14:08,276 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:14:08,276 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:08,277 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,277 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:14:08,277 DEBUG SenderThread:1741299 [sender.py:send():379] send: summary +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:14:08,278 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:14:08,278 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,279 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:14:08,279 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:14:08,279 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:08,279 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,279 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:14:08,279 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:14:08,880 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:09,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:09,419 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:14:09,419 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:09,420 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:09,420 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:14:09,420 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:09,420 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:14:09,420 INFO SenderThread:1741299 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:14:09,422 INFO SenderThread:1741299 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:14:09,436 INFO SenderThread:1741299 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:14:09,453 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:14:09,453 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:09,453 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:14:09,453 DEBUG SenderThread:1741299 [sender.py:send():379] send: artifact +2024-12-31 21:14:09,880 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:09,905 INFO wandb-upload_1:1741299 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi6tvm6k6 +2024-12-31 21:14:09,906 INFO wandb-upload_0:1741299 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpejq8o616 +2024-12-31 21:14:10,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:10,107 INFO wandb-upload_2:1741299 [upload_job.py:push():88] Uploaded file /tmp/tmp07nyr7fc/wandb-job.json +2024-12-31 21:14:10,656 INFO SenderThread:1741299 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExODg5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDExNTMxNA==', 'versionIndex': 390}}} +2024-12-31 21:14:10,657 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:10,657 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:14:10,657 INFO SenderThread:1741299 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml config.yaml +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json wandb-summary.json +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt requirements.txt +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log output.log +2024-12-31 21:14:10,882 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch diff.patch +2024-12-31 21:14:10,888 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:14:10,888 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:14:10,888 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:10,889 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:10,889 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:14:10,889 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:10,889 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:14:10,889 INFO SenderThread:1741299 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:14:11,055 INFO wandb-upload_1:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:11,076 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,076 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,081 INFO wandb-upload_3:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:14:11,119 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:11,122 INFO wandb-upload_0:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt +2024-12-31 21:14:11,322 INFO Thread-11 (_thread_body):1741299 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:14:11,322 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,322 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:14:11,323 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:14:11,323 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,323 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:14:11,323 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:14:11,389 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,389 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:14:11,389 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:14:11,390 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:14:11,390 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,390 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send():379] send: final +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send():379] send: footer +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,390 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:14:11,391 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,392 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:14:11,393 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:14:11,393 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:11,446 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:14:11,447 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:14:11,447 INFO HandlerThread:1741299 [handler.py:finish():882] shutting down handler +2024-12-31 21:14:12,392 INFO WriterThread:1741299 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [sender.py:finish():1615] shutting down sender +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [file_pusher.py:join():175] waiting for file pusher diff --git a/ksj6ie6d/wandb/debug.log b/ksj6ie6d/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d14b68a63ac59986f690078fdb81ecd9dea2febf --- /dev/null +++ b/ksj6ie6d/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Configure stats pid to 1741133 +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/settings +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug.log +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug-internal.log +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d'} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():619] starting backend +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():623] setting up manager +2024-12-31 21:08:37,536 INFO MainThread:1741133 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:08:37,537 INFO MainThread:1741133 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:08:37,550 INFO MainThread:1741133 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:08:37,556 INFO MainThread:1741133 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:08:37,801 INFO MainThread:1741133 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:08:37,871 INFO MainThread:1741133 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:08:37,871 INFO MainThread:1741133 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:08:38,195 INFO MainThread:1741133 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:08:38,195 INFO MainThread:1741133 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/tb, True +2024-12-31 21:10:24,810 INFO MainThread:1741133 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:14:12,448 WARNING MsgRouterThr:1741133 [router.py:message_loop():77] message_loop has been closed diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45600d6224842349e5a9ab4bf7c7b5e36a751b4e --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml @@ -0,0 +1,894 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735697317 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..469b7cb24c86725dac8897715810ce5b4ce34a80 --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch @@ -0,0 +1,85 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 3b7c93f..0f6a9ec 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -129,7 +129,12 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1051_0' # all + + # RGBD+mask+act history +-wandb_run_id: '1000_0' # all ++# wandb_run_id: '1000_0' # all ++ ++# all books, 10/20 demos per book ++# RGBD+mask+act history ++wandb_run_id: '1007_0' ++ + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +@@ -174,7 +179,7 @@ residual_weight: '${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name + + # old_experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${name_of_expert_demo}_valgroups_${val_num_groups}/${old_feature_type}/${seed}/${agent.name}/${wandb_run_id} + experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +-final_experiment_dir: ${experiment_dir}/${oc.generate_run_id} ++final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} + # ${now:%H%M%S} + hydra: + run: +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 5ed3eb4..21e335c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -148,7 +148,14 @@ remote_host = 'grasp-login1' + # ] + + run_id_list = [ +- '1000_0', ++ # '1002_0', ++ # '1009_1', ++ # '1009_2', ++ # '1022_0', ++ # '1022_0', ++ # '1023_0', ++ # '1007_0', ++ '1007_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 3482538..531c122 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -463,6 +463,8 @@ class Workspace: + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + ++ table_data = [table_data] ++ + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) +@@ -471,11 +473,13 @@ class Workspace: + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ +- "jobType": "eval", ++ "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, +- "state": "finished" ++ "state": "finished", ++ "config.grasped_obj_name": grasped_obj_name, ++ "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -503,6 +507,8 @@ class Workspace: + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub ++ ++ total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..63c0819f97c3c939d5defff5918fc48a3a717611 --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log @@ -0,0 +1,37 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735697424.812252]: resetting environment +[INFO] [1735697424.815678]: cleared current plan +[INFO] [1735697424.816547]: moving to home +[INFO] [1735697425.819544]: reached home +[INFO] [1735697425.821156]: reset action history +[INFO] [1735697428.528793]: environment reset +Starting episode 0 +[INFO] [1735697428.530769]: resetting environment +[INFO] [1735697428.534760]: cleared current plan +[INFO] [1735697428.535788]: moving to home +[INFO] [1735697429.537259]: reached home +[INFO] [1735697429.539011]: reset action history +[INFO] [1735697432.253031]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1735697641.368143]: Saving video... +Error executing job with overrides: ['agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'] +Traceback (most recent call last): + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 596, in main + @hydra.main(config_path='cfgs', config_name='config_eval') + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 320, in eval + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 607, in wait + signaled = self._cond.wait(timeout) + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 320, in wait + waiter.acquire() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/core.py", line 610, in _ros_signal + prev_handler(sig, stackframe) + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 146, in signal_handler + video_filepath = self.video_recorder.save() + File "/home/leonmkim/fish_leon/FISH/video.py", line 110, in save + self.recording_timer.shutdown() +AttributeError: 'VideoRecorder' object has no attribute 'recording_timer' +Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace. \ No newline at end of file diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..01c5f990d0e45adcc9a1a9482048f0948d32b627 --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T02:08:37.895012", + "startedAt": "2025-01-01T02:08:37.529957", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "284407f87cd673789a10dbd8e4d4f41dcad30ded" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.7145, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4481.251, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.919, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 500.70812606811523 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a8584a2aa4bcb71c8f3916fb1e176b537e4fe398 --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 324}} \ No newline at end of file diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug-internal.log b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3c70aa967d479378de23bdd5eb2165bd508bd9a --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug-internal.log @@ -0,0 +1,664 @@ +2024-12-31 21:08:37,537 INFO StreamThr :1741299 [internal.py:wandb_internal():85] W&B internal server running at pid: 1741299, started at: 2024-12-31 21:08:37.536993 +2024-12-31 21:08:37,538 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:08:37,550 INFO WriterThread:1741299 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb +2024-12-31 21:08:37,551 DEBUG SenderThread:1741299 [sender.py:send():379] send: header +2024-12-31 21:08:37,559 DEBUG SenderThread:1741299 [sender.py:send():379] send: run +2024-12-31 21:08:37,786 INFO SenderThread:1741299 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files +2024-12-31 21:08:37,786 INFO SenderThread:1741299 [sender.py:_start_run_threads():1188] run started: ksj6ie6d with start time 1735697317.537409 +2024-12-31 21:08:37,802 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:08:37,802 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:08:37,875 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:08:37,880 DEBUG HandlerThread:1741299 [system_info.py:__init__():26] System info init +2024-12-31 21:08:37,880 DEBUG HandlerThread:1741299 [system_info.py:__init__():41] System info init done +2024-12-31 21:08:37,880 INFO HandlerThread:1741299 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:08:37,881 INFO SystemMonitor:1741299 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:08:37,881 INFO HandlerThread:1741299 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:08:37,881 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:08:37,882 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:08:37,882 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:08:37,883 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:08:37,883 INFO SystemMonitor:1741299 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:08:37,894 DEBUG HandlerThread:1741299 [system_info.py:probe():152] Probing system +2024-12-31 21:08:37,896 DEBUG HandlerThread:1741299 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:probe():200] Probing system done +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:08:37.895012', 'startedAt': '2025-01-01T02:08:37.529957', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.7145, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4481.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.919, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 500.70812606811523}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:08:37,901 INFO HandlerThread:1741299 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:08:37,901 INFO HandlerThread:1741299 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:08:37,901 DEBUG HandlerThread:1741299 [system_info.py:_save_code():44] Saving code +2024-12-31 21:08:37,903 DEBUG HandlerThread:1741299 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:08:37,903 DEBUG HandlerThread:1741299 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:08:37,964 DEBUG HandlerThread:1741299 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:08:37,964 INFO HandlerThread:1741299 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:08:37,967 DEBUG SenderThread:1741299 [sender.py:send():379] send: files +2024-12-31 21:08:37,967 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:08:37,967 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:08:37,968 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:08:38,193 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:08:38,193 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:08:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:08:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:38,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:08:38,237 INFO HandlerThread:1741299 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/tb" + save: true +} +control { + relay_id: "7b45fda39ce243519ba995b8b7d508f7" +} +_info { + stream_id: "ksj6ie6d" +} + +2024-12-31 21:08:38,237 DEBUG HandlerThread:1741299 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:08:38,261 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/323z6vdp-diff.patch +2024-12-31 21:08:38,265 INFO wandb-upload_0:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/lk0pmb1b-wandb-metadata.json +2024-12-31 21:08:38,283 INFO wandb-upload_1:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/noqhmg2m-code/FISH/eval_robot.py +2024-12-31 21:08:38,310 DEBUG SenderThread:1741299 [sender.py:send():379] send: telemetry +2024-12-31 21:08:38,310 DEBUG SenderThread:1741299 [sender.py:send():379] send: tbrecord +2024-12-31 21:08:38,311 DEBUG SenderThread:1741299 [sender.py:send():379] send: files +2024-12-31 21:08:38,311 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 with policy live +2024-12-31 21:08:38,502 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /tmp/tmp7ueklbcbwandb/l588opbr-events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH/eval_robot.py +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:08:38,788 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch +2024-12-31 21:08:38,789 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code +2024-12-31 21:08:38,789 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH +2024-12-31 21:08:39,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:40,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:40,787 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:08:41,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:42,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:43,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:43,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:44,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:45,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:46,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:47,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:48,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:48,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:49,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:50,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:51,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:52,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:08:53,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:08:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:53,264 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:54,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:55,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:56,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:57,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:58,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:08:59,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:08:59,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:00,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:01,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:02,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:03,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:04,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:04,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:05,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:06,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:07,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:08,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:08,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:09,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:09,324 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:09,796 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:09:10,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:11,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:12,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:13,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:14,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:15,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:15,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:16,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:17,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:18,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:19,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:20,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:20,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:21,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:22,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:23,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:23,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:24,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:25,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:25,343 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:26,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:27,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:28,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:29,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:30,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:31,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:31,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:32,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:33,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:34,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:35,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:36,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:36,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:37,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:37,884 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:09:37,884 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:09:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:38,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:39,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:40,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:41,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:41,334 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:42,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:43,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:44,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:45,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:46,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:47,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:47,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:48,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:49,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:50,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:51,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:52,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:52,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:09:53,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:09:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:54,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:55,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:56,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:57,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:57,282 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:09:58,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:09:59,204 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:00,204 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:01,206 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:02,206 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:03,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:03,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:04,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:05,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:06,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:07,207 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:07,885 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:10:08,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:08,194 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:08,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:08,282 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:09,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:10,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:11,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:12,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:13,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:14,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:14,208 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:15,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:16,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:17,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:18,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:19,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:20,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:20,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:21,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:22,209 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:23,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:23,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:24,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:24,811 DEBUG SenderThread:1741299 [sender.py:send():379] send: config +2024-12-31 21:10:25,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:25,820 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:26,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:26,818 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:27,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:28,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:29,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:30,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:30,819 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:31,210 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:31,581 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:32,211 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:33,211 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:34,212 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:34,821 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:10:35,212 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:36,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:37,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:37,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:37,885 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:10:38,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:38,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:39,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:40,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:41,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:42,213 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:42,281 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:43,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:44,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:45,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:46,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:47,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:48,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:48,214 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:48,825 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:10:49,215 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:50,215 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:51,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:52,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:53,194 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:10:53,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:10:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:53,355 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:54,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:55,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:56,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:57,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:58,216 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:10:59,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:10:59,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:00,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:01,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:02,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:03,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:04,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:04,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:05,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:06,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:07,217 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:07,886 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:11:08,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:08,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:09,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:09,340 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:10,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:11,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:12,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:13,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:14,218 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:15,198 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:15,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:16,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:17,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:18,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:19,219 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:20,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:20,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:21,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:22,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:23,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:23,195 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:24,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:25,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:25,272 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:26,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:27,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:28,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:29,220 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:30,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:31,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:31,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:32,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:33,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:34,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:35,221 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:36,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:37,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:37,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:37,886 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:11:38,195 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:38,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:39,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:40,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:41,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:42,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:42,275 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:43,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:44,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:45,222 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:46,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:47,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:48,199 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:48,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:49,223 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:50,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:51,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:52,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:53,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:11:53,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:11:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:53,320 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:54,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:55,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:56,224 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:57,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:58,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:11:59,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:11:59,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:00,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:01,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:02,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:03,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:04,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:04,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:05,225 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:06,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:07,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:07,887 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:12:08,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:08,196 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:09,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:09,357 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:10,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:11,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:12,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:13,226 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:14,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:15,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:15,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:16,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:17,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:18,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:19,227 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:20,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:20,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:21,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:22,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:23,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:23,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:23,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:24,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:25,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:25,277 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:26,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:27,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:28,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:29,228 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:30,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:31,200 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:31,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:32,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:33,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:34,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:35,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:36,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:36,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:37,229 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:37,887 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:12:38,196 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:38,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:38,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:39,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:40,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:41,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:41,308 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:42,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:43,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:44,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:45,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:46,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:47,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:47,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:48,230 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:49,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:50,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:51,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:52,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:52,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:53,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:12:53,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:12:53,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:54,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:55,231 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:56,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:57,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:57,284 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:12:58,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:12:59,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:00,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:01,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:02,232 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:03,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:03,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:04,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:05,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:06,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:07,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:07,888 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:13:08,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:08,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:08,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:08,270 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:09,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:10,233 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:11,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:12,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:13,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:14,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:14,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:15,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:16,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:17,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:18,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:19,201 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:19,234 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:20,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:21,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:22,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:23,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:23,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:23,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:24,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:24,265 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:25,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:26,235 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:27,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:28,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:29,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:30,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:30,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:31,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:32,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:33,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:34,236 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:35,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:35,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:36,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:37,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:37,888 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:13:38,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:38,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:38,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:39,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:40,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:40,318 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:41,237 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:42,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:43,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:44,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:45,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:46,202 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:46,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:47,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:48,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:49,238 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:50,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:51,203 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:51,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:52,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:53,197 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:13:53,197 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:13:53,241 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:54,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:55,239 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:56,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:56,268 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:13:57,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:58,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:13:59,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:00,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:00,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:01,240 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:01,368 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:02,074 DEBUG SenderThread:1741299 [sender.py:send():379] send: exit +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:send_exit():586] handling exit code: 1 +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:send_exit():588] handling runtime: 324 +2024-12-31 21:14:02,074 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_exit():594] send defer +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,075 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:14:02,075 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,075 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:14:02,075 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:14:02,075 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:14:02,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:14:02,076 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:14:02,076 DEBUG SystemMonitor:1741299 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:14:02,076 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:14:02,129 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:14:02,130 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,130 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:14:02,130 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:14:02,130 DEBUG SenderThread:1741299 [sender.py:send():379] send: stats +2024-12-31 21:14:02,130 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,130 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:14:02,131 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:02,131 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:14:02,131 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:14:02,131 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:02,131 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:14:02,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:02,878 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:03,879 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:08,276 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:14:08,276 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,276 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:14:08,276 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:14:08,276 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:08,277 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,277 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:14:08,277 DEBUG SenderThread:1741299 [sender.py:send():379] send: summary +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:14:08,278 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:14:08,278 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:14:08,278 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,279 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:14:08,279 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:14:08,279 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:14:08,279 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:08,279 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:14:08,279 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:08,279 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:14:08,880 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:09,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:09,419 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:14:09,419 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:09,420 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:09,420 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:14:09,420 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:09,420 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:14:09,420 INFO SenderThread:1741299 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:14:09,422 INFO SenderThread:1741299 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:14:09,436 INFO SenderThread:1741299 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:14:09,453 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:14:09,453 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:09,453 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:14:09,453 DEBUG SenderThread:1741299 [sender.py:send():379] send: artifact +2024-12-31 21:14:09,880 INFO Thread-12 :1741299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:09,905 INFO wandb-upload_1:1741299 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi6tvm6k6 +2024-12-31 21:14:09,906 INFO wandb-upload_0:1741299 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpejq8o616 +2024-12-31 21:14:10,075 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:10,107 INFO wandb-upload_2:1741299 [upload_job.py:push():88] Uploaded file /tmp/tmp07nyr7fc/wandb-job.json +2024-12-31 21:14:10,656 INFO SenderThread:1741299 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExODg5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDExNTMxNA==', 'versionIndex': 390}}} +2024-12-31 21:14:10,657 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:10,657 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:14:10,657 INFO SenderThread:1741299 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml config.yaml +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json wandb-summary.json +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 events.out.tfevents.1735697318.leonmkim-ROG-Strix-G15CS-G15CS.1741133.0 +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt requirements.txt +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:14:10,881 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log output.log +2024-12-31 21:14:10,882 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/diff.patch diff.patch +2024-12-31 21:14:10,888 INFO SenderThread:1741299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:14:10,888 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:14:10,888 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:10,889 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:10,889 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:14:10,889 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:10,889 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:14:10,889 INFO SenderThread:1741299 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:14:11,055 INFO wandb-upload_1:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/wandb-summary.json +2024-12-31 21:14:11,076 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,076 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,081 INFO wandb-upload_3:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/config.yaml +2024-12-31 21:14:11,119 INFO wandb-upload_2:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/output.log +2024-12-31 21:14:11,122 INFO wandb-upload_0:1741299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/files/requirements.txt +2024-12-31 21:14:11,322 INFO Thread-11 (_thread_body):1741299 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:14:11,322 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,322 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:14:11,323 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:14:11,323 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,323 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:14:11,323 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:14:11,323 INFO SenderThread:1741299 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:14:11,389 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,389 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:14:11,389 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,389 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:14:11,390 INFO SenderThread:1741299 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:14:11,390 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:14:11,390 INFO HandlerThread:1741299 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send():379] send: final +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send():379] send: footer +2024-12-31 21:14:11,390 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: defer +2024-12-31 21:14:11,390 INFO SenderThread:1741299 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:14:11,391 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:14:11,391 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,392 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:14:11,393 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:14:11,393 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:14:11,446 DEBUG SenderThread:1741299 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:14:11,447 INFO MainThread:1741299 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:14:11,447 DEBUG HandlerThread:1741299 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:14:11,447 INFO HandlerThread:1741299 [handler.py:finish():882] shutting down handler +2024-12-31 21:14:12,392 INFO WriterThread:1741299 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [sender.py:finish():1615] shutting down sender +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:14:12,447 INFO SenderThread:1741299 [file_pusher.py:join():175] waiting for file pusher diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug.log b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d14b68a63ac59986f690078fdb81ecd9dea2febf --- /dev/null +++ b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Configure stats pid to 1741133 +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:08:37,531 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/settings +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug.log +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/logs/debug-internal.log +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d'} +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():619] starting backend +2024-12-31 21:08:37,532 INFO MainThread:1741133 [wandb_init.py:init():623] setting up manager +2024-12-31 21:08:37,536 INFO MainThread:1741133 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:08:37,537 INFO MainThread:1741133 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:08:37,550 INFO MainThread:1741133 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:08:37,556 INFO MainThread:1741133 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:08:37,801 INFO MainThread:1741133 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:08:37,871 INFO MainThread:1741133 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:08:37,871 INFO MainThread:1741133 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:08:38,194 INFO MainThread:1741133 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:08:38,195 INFO MainThread:1741133 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:08:38,195 INFO MainThread:1741133 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ksj6ie6d/tb, True +2024-12-31 21:10:24,810 INFO MainThread:1741133 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:14:12,448 WARNING MsgRouterThr:1741133 [router.py:message_loop():77] message_loop has been closed diff --git a/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8cc30a84389e6a1dbfa9be9586613a9f33950957 Binary files /dev/null and b/ksj6ie6d/wandb/run-20241231_210837-ksj6ie6d/run-ksj6ie6d.wandb differ diff --git a/nkdxms1x/.hydra/config.yaml b/nkdxms1x/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/nkdxms1x/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nkdxms1x/.hydra/hydra.yaml b/nkdxms1x/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56f6c383d91f28096610b9f4f3a925ba7be3e37c --- /dev/null +++ b/nkdxms1x/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nkdxms1x/.hydra/overrides.yaml b/nkdxms1x/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nkdxms1x/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nkdxms1x/episode_rosbags/aligned_depth_to_color_K.npy b/nkdxms1x/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nkdxms1x/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nkdxms1x/episode_rosbags/cam_tf_world.npy b/nkdxms1x/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nkdxms1x/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nkdxms1x/episode_rosbags/color_K.npy b/nkdxms1x/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nkdxms1x/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nkdxms1x/episode_rosbags/depth_K.npy b/nkdxms1x/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nkdxms1x/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nkdxms1x/episode_rosbags/episode_0_2024-12-31-21-03-48.bag b/nkdxms1x/episode_rosbags/episode_0_2024-12-31-21-03-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..ef2da099b770e37b7cb18393565b22836b8b5cf9 --- /dev/null +++ b/nkdxms1x/episode_rosbags/episode_0_2024-12-31-21-03-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13aa7dbab205239b9538e7400d8f1a0c9622be81167c452f3b75501f534eac47 +size 764896126 diff --git a/nkdxms1x/episode_rosbags/episode_1_2024-12-31-21-04-39.bag b/nkdxms1x/episode_rosbags/episode_1_2024-12-31-21-04-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..5b0d893e23a3dac8152996a90b0b630dd39e752b --- /dev/null +++ b/nkdxms1x/episode_rosbags/episode_1_2024-12-31-21-04-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52307addc6c4c6bb279cfa50662bfd71b3fa39735628faa620a59d57d1a9c8d3 +size 670611766 diff --git a/nkdxms1x/eval_robot.log b/nkdxms1x/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..f39dee5bc3161f1ab5c6c0adbff0d049fa0efd51 --- /dev/null +++ b/nkdxms1x/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 21:03:17,491][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:590: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:03:17,494][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:526: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nkdxms1x/eval_video/0_eval.mp4 b/nkdxms1x/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3a5caae9d785a885dc8fafbc3b1bc8dc899a6ba --- /dev/null +++ b/nkdxms1x/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e739aa3a1064d7a115c648ec192bb1b5b34388ac6d31936a7ef6fd1ca52ae4ea +size 678402 diff --git a/nkdxms1x/eval_video/1_eval.mp4 b/nkdxms1x/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c40124fc43aa9234a041f0c5b2f73630fee63d3 --- /dev/null +++ b/nkdxms1x/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a45cc58516bfa8280677b9bd2af88e662726e4efdf25b8e31cbbf385486eed +size 594378 diff --git a/nkdxms1x/eval_video/2_eval.mp4 b/nkdxms1x/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68ac988a741e39e6fbb27b273d0a97a784b5d644 --- /dev/null +++ b/nkdxms1x/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1c0520eab44850309611a44e67873cd28908f9005d185df5b339ad1874bcd7 +size 1157154 diff --git a/nkdxms1x/eval_video/3_eval.mp4 b/nkdxms1x/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd6a24d715fd9d7fcb2bd344eadeef7ba9074b5f --- /dev/null +++ b/nkdxms1x/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e47918912079386856849116782c06f1a9ea624d9c21c42d468e6a4df4f115 +size 1157639 diff --git a/nkdxms1x/eval_video/4_eval.mp4 b/nkdxms1x/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68b9df691aebf9b8fa67c459c85760cacd28d802 --- /dev/null +++ b/nkdxms1x/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43d65e2ea74e852f815b6bdd0071163c9bd16ea1f6cadc977d529abe360e33d +size 1171802 diff --git a/nkdxms1x/tb/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 b/nkdxms1x/tb/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 new file mode 100644 index 0000000000000000000000000000000000000000..bcde6c8ccb855a91e55f9bee981ac6f58dbff7ca --- /dev/null +++ b/nkdxms1x/tb/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c90d15299e2b416f545fd8c0ec789adc54fe4a8b276eaaeba07705e41a820e62 +size 5483 diff --git a/nkdxms1x/wandb/debug-internal.log b/nkdxms1x/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ffb40e62e27cf598ed7ad2d484ab9ecfd7d6037a --- /dev/null +++ b/nkdxms1x/wandb/debug-internal.log @@ -0,0 +1,807 @@ +2024-12-31 21:03:23,435 INFO StreamThr :1740211 [internal.py:wandb_internal():85] W&B internal server running at pid: 1740211, started at: 2024-12-31 21:03:23.435568 +2024-12-31 21:03:23,436 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:03:23,449 INFO WriterThread:1740211 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb +2024-12-31 21:03:23,450 DEBUG SenderThread:1740211 [sender.py:send():379] send: header +2024-12-31 21:03:23,459 DEBUG SenderThread:1740211 [sender.py:send():379] send: run +2024-12-31 21:03:23,694 INFO SenderThread:1740211 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files +2024-12-31 21:03:23,694 INFO SenderThread:1740211 [sender.py:_start_run_threads():1188] run started: nkdxms1x with start time 1735697003.437517 +2024-12-31 21:03:23,711 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:03:23,711 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:03:23,755 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:03:23,760 DEBUG HandlerThread:1740211 [system_info.py:__init__():26] System info init +2024-12-31 21:03:23,760 DEBUG HandlerThread:1740211 [system_info.py:__init__():41] System info init done +2024-12-31 21:03:23,760 INFO HandlerThread:1740211 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:03:23,760 INFO SystemMonitor:1740211 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:03:23,761 INFO HandlerThread:1740211 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:03:23,762 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:03:23,762 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:03:23,773 DEBUG HandlerThread:1740211 [system_info.py:probe():152] Probing system +2024-12-31 21:03:23,774 DEBUG HandlerThread:1740211 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:probe():200] Probing system done +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:03:23.773340', 'startedAt': '2025-01-01T02:03:23.425372', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.004875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.995, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 495.41462326049805}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:03:23,779 INFO HandlerThread:1740211 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:03:23,779 INFO HandlerThread:1740211 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:_save_code():44] Saving code +2024-12-31 21:03:23,780 DEBUG HandlerThread:1740211 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:03:23,781 DEBUG HandlerThread:1740211 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:03:23,842 DEBUG HandlerThread:1740211 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:03:23,842 INFO HandlerThread:1740211 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:03:23,845 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:03:23,845 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:03:23,846 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:03:23,846 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:03:24,088 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:03:24,088 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:03:24,089 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/q79k8vpc-code/FISH/eval_robot.py +2024-12-31 21:03:24,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:24,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:24,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:24,091 INFO HandlerThread:1740211 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/tb" + save: true +} +control { + relay_id: "ee70b23f69f642a48cfae1d64e8e8ef4" +} +_info { + stream_id: "nkdxms1x" +} + +2024-12-31 21:03:24,091 DEBUG HandlerThread:1740211 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:03:24,094 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/e8mji6dz-wandb-metadata.json +2024-12-31 21:03:24,107 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ti5udqhq-diff.patch +2024-12-31 21:03:24,159 DEBUG SenderThread:1740211 [sender.py:send():379] send: telemetry +2024-12-31 21:03:24,159 DEBUG SenderThread:1740211 [sender.py:send():379] send: tbrecord +2024-12-31 21:03:24,160 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:03:24,160 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 with policy live +2024-12-31 21:03:24,345 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/xguvxaej-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH +2024-12-31 21:03:24,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code +2024-12-31 21:03:25,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:26,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:26,695 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:27,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:28,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:29,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:29,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:29,760 DEBUG SenderThread:1740211 [sender.py:send():379] send: config +2024-12-31 21:03:30,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:31,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:32,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:32,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:33,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:34,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:34,495 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:34,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:35,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:36,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:36,698 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:37,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:38,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:38,699 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:39,088 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:39,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:39,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:40,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:40,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:41,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:42,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:43,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:44,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:45,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:45,988 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:46,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:46,701 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:47,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:48,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:48,701 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:49,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:50,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:50,702 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:51,015 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:51,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:52,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:52,704 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:53,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:54,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:54,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:54,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:54,704 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:55,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:56,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:56,178 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:56,705 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:03:56,705 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:57,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:58,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:58,706 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:59,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:00,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:00,706 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:01,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:01,508 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:02,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:02,707 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:03,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:04,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:04,559 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:04:04,559 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 with policy now +2024-12-31 21:04:04,561 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:04:04,562 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:04:04,562 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:04:04,562 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:04:04,707 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos +2024-12-31 21:04:04,797 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/uj3y4dh9-media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:05,052 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/cgpichjt-media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:05,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:06,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:06,573 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:06,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:07,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:08,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:09,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:09,091 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:09,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:10,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:10,709 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:11,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:12,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:12,533 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:12,710 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:13,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:14,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:15,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:16,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:17,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:18,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:18,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:19,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:20,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:21,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:22,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:23,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:23,762 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:04:23,763 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:04:23,764 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:24,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:24,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:24,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:25,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:26,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:27,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:28,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:29,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:29,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:30,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:31,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:32,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:33,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:34,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:34,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:35,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:36,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:36,718 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:04:37,005 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/xgxz2wki-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:04:37,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:38,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:38,719 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:39,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:39,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:39,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:39,237 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:40,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:41,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:42,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:42,720 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:43,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:44,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:44,407 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:44,721 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:45,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:46,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:46,722 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:47,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:48,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:48,723 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:49,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:49,414 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:50,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:50,723 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:51,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:52,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:52,724 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:53,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:53,763 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:04:54,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:54,092 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:54,118 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:54,327 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:04:54,327 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 with policy now +2024-12-31 21:04:54,331 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:04:54,332 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:04:54,332 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:04:54,332 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:04:54,567 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/m9tsz8o4-media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:54,725 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:04:54,725 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:54,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:54,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:04:54,973 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/apzt3djp-media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:55,117 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:55,354 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:56,115 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:56,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:57,116 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:58,120 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:58,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:59,126 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:00,127 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:00,627 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:00,727 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:01,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:02,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:02,728 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:03,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:04,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:05,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:06,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:06,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:07,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:08,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:09,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:09,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:09,133 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:10,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:11,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:11,211 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:12,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:13,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:14,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:15,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:16,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:17,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:17,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:18,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:19,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:19,501 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:19,501 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:19,502 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:19,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:19,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:05:20,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:20,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:21,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:22,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:22,269 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:22,735 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:23,144 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:23,764 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:05:24,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:24,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:24,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:24,736 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:25,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:26,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:26,737 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:27,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:27,441 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:28,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:28,738 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:29,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:30,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:30,738 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:31,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:32,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:32,448 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:32,739 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:33,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:34,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:34,740 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:35,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:36,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:36,741 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:37,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:37,469 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:38,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:38,741 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:39,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:39,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:39,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:40,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:40,742 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:41,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:42,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:42,642 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:42,743 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:43,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:44,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:44,744 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:45,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:46,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:46,744 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:47,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:47,650 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:48,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:48,745 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:49,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:50,142 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:50,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:50,924 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:05:50,924 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 with policy now +2024-12-31 21:05:50,924 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:05:50,925 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:50,925 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:50,925 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:51,143 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:51,174 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ueb0i9jh-media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:51,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:51,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:51,747 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:05:52,054 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/1ppz161i-media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:52,141 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:52,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:52,959 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:53,146 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:53,764 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:05:54,102 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:54,102 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:54,149 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:55,147 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:56,145 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:56,748 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:57,145 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:58,146 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:58,393 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:58,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:59,148 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:59,329 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:59,330 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:59,330 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:59,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:59,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:05:59,980 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/iaw5gjma-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:06:00,149 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:00,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:01,151 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:02,151 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:03,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:03,471 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:04,165 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:04,751 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:05,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:06,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:06,752 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:07,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:08,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:08,502 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:08,752 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:09,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:09,095 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:09,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:10,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:10,753 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:11,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:12,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:12,754 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:13,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:13,697 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:14,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:14,755 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:15,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:16,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:16,755 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:17,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:18,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:18,698 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:18,756 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:19,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:20,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:20,757 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:21,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:22,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:22,758 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:23,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:23,702 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:23,765 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:06:24,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:24,095 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:24,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:24,758 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:25,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:26,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:26,759 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:27,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:28,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:28,760 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:28,906 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:29,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:30,173 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:30,760 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:31,152 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:06:31,153 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a0e47918912079386856.mp4 with policy now +2024-12-31 21:06:31,156 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:06:31,156 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:06:31,156 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:06:31,157 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:06:31,171 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:31,412 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/tjltem0s-media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:06:32,054 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/a1eck4xq-media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:32,185 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:32,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:33,180 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:34,107 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:34,178 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:35,180 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:36,185 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:36,763 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:37,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:38,181 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:38,763 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:39,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:39,098 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:39,189 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:39,210 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:40,183 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:41,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:42,188 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:43,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:44,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:45,106 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:45,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:46,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:47,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:47,766 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:06:48,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:48,388 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:06:48,388 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:06:48,389 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:06:48,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:06:48,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:49,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:50,122 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:50,197 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:50,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:51,196 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:52,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:52,768 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:53,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:53,769 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:06:54,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:54,096 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:54,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:54,770 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:55,133 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:55,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:56,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:56,770 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:57,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:58,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:58,771 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:59,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:00,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:00,309 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:00,772 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:01,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:02,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:02,773 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:03,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:04,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:04,773 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:05,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:05,326 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:06,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:06,774 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:07,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:08,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:08,775 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:09,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:09,096 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:09,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:10,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:10,516 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:10,776 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:11,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:12,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:12,777 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:13,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:14,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:14,777 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:15,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:15,719 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:16,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:16,778 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:17,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:18,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:18,779 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:19,201 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:19,953 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:19,953 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 with policy now +2024-12-31 21:07:19,956 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:19,956 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:19,956 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:19,957 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:20,199 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:20,214 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/n4g10zrq-media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:07:20,976 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:21,048 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/u21i1vsf-media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:21,201 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:22,220 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:22,781 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:23,206 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:23,769 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:07:24,113 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:24,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:24,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:24,781 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:25,223 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:26,231 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:26,249 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:27,207 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:28,208 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:28,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:28,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,021 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/a07y7rzg-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:29,245 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:07:29,245 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:07:29,477 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:29,699 INFO wandb-upload_1:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo_cu9z3a +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media +2024-12-31 21:07:29,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval +2024-12-31 21:07:29,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table +2024-12-31 21:07:29,830 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:07:29,842 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:30,102 INFO SenderThread:1740211 [sender.py:send_request_log_artifact():1525] logged artifact run-nkdxms1x-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTA3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2NjU0NQ==', 'latestArtifact': None}} +2024-12-31 21:07:30,104 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:30,104 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2024-12-31 21:07:30,105 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,105 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,106 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,107 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,107 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,108 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,108 DEBUG SenderThread:1740211 [sender.py:send():379] send: telemetry +2024-12-31 21:07:30,108 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,108 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,109 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,109 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,109 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,109 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,111 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,111 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,112 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,112 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,112 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,112 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,113 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,113 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:07:30,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:30,277 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ly18bboo-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:30,497 INFO wandb-upload_1:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvrm_yjbf +2024-12-31 21:07:30,526 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/bhwrc5ru-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval +2024-12-31 21:07:30,850 INFO SenderThread:1740211 [sender.py:send_request_log_artifact():1525] logged artifact run-nkdxms1x-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTA4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2NjU0Nw==', 'latestArtifact': None}} +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:30,851 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json with policy now +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,851 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:31,032 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/089q6my2-media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:31,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:31,303 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/leshttye-media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:31,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:31,852 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:32,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:33,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:34,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:35,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:36,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:37,118 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:37,219 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:37,786 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:07:38,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:39,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:39,097 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:39,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:40,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:41,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:42,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:43,069 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:43,110 DEBUG SenderThread:1740211 [sender.py:send():379] send: exit +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():588] handling runtime: 259 +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():594] send defer +2024-12-31 21:07:43,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,110 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:07:43,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:07:43,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:07:43,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:07:43,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:07:43,111 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:07:43,112 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:07:43,112 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:07:43,112 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:07:43,163 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:07:43,164 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:07:43,164 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:07:43,164 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,164 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:07:43,164 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:07:43,164 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:07:43,164 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,165 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:07:43,165 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,165 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:07:43,165 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:07:43,165 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,165 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:07:43,789 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:44,789 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,444 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:07:49,444 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:49,444 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:49,444 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:07:49,445 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:49,446 DEBUG SenderThread:1740211 [sender.py:send():379] send: summary +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:49,446 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:07:49,446 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,446 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:07:49,447 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:07:49,447 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:49,447 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,447 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:07:49,447 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:07:49,791 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:50,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:07:52,389 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:52,389 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:52,389 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:07:52,389 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:07:52,392 INFO SenderThread:1740211 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:07:52,393 INFO SenderThread:1740211 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:07:52,404 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:07:52,404 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:52,404 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:07:52,404 DEBUG SenderThread:1740211 [sender.py:send():379] send: artifact +2024-12-31 21:07:52,753 INFO wandb-upload_1:1740211 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaag6j_j3 +2024-12-31 21:07:52,793 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:52,941 INFO wandb-upload_0:1740211 [upload_job.py:push():88] Uploaded file /tmp/tmp3rvw6g5e/wandb-job.json +2024-12-31 21:07:52,942 INFO wandb-upload_2:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8_hforgk +2024-12-31 21:07:53,112 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTMxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDExMjAxNQ==', 'versionIndex': 389}}} +2024-12-31 21:07:53,477 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:07:53,793 INFO SenderThread:1740211 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml config.yaml +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json wandb-summary.json +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt requirements.txt +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log output.log +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch diff.patch +2024-12-31 21:07:53,796 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:07:53,801 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:07:53,803 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:53,803 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:53,803 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:07:53,804 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:53,804 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:07:53,804 INFO SenderThread:1740211 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:07:53,970 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:54,008 INFO wandb-upload_3:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt +2024-12-31 21:07:54,029 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:07:54,031 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/3jni8frz-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:54,072 INFO wandb-upload_4:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:54,113 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,272 INFO Thread-11 (_thread_body):1740211 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:07:54,272 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,272 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:07:54,273 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,273 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:07:54,273 INFO SenderThread:1740211 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:07:54,274 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:07:54,274 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,274 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:07:54,274 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,275 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:07:54,275 INFO SenderThread:1740211 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:07:54,347 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,347 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:07:54,347 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:07:54,348 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,348 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send():379] send: final +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send():379] send: footer +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,348 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:07:54,348 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,349 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,349 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:07:54,351 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:07:54,351 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:54,401 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,401 INFO MainThread:1740211 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:07:54,402 INFO MainThread:1740211 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:07:54,402 INFO MainThread:1740211 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:07:54,402 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:07:54,402 INFO HandlerThread:1740211 [handler.py:finish():882] shutting down handler +2024-12-31 21:07:55,352 INFO WriterThread:1740211 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [sender.py:finish():1615] shutting down sender +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [file_pusher.py:join():175] waiting for file pusher diff --git a/nkdxms1x/wandb/debug.log b/nkdxms1x/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d085c16e63e39bc828b3777514788f522d4338fe --- /dev/null +++ b/nkdxms1x/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Configure stats pid to 1740046 +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/settings +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug.log +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug-internal.log +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x'} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():619] starting backend +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():623] setting up manager +2024-12-31 21:03:23,434 INFO MainThread:1740046 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:03:23,437 INFO MainThread:1740046 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:03:23,449 INFO MainThread:1740046 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:03:23,457 INFO MainThread:1740046 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:03:23,710 INFO MainThread:1740046 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:03:23,750 INFO MainThread:1740046 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:03:23,750 INFO MainThread:1740046 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:03:24,090 INFO MainThread:1740046 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:03:24,091 INFO MainThread:1740046 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/tb, True +2024-12-31 21:03:29,760 INFO MainThread:1740046 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:07:55,403 WARNING MsgRouterThr:1740046 [router.py:message_loop():77] message_loop has been closed diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..531c122414c0ae9ed13049add4de27fe555222b1 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py @@ -0,0 +1,599 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + "config.grasped_obj_name": grasped_obj_name, + "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fadb9783ea921cdc47967a0919c1c8c5cfdf4276 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735697003 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..469b7cb24c86725dac8897715810ce5b4ce34a80 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch @@ -0,0 +1,85 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 3b7c93f..0f6a9ec 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -129,7 +129,12 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1051_0' # all + + # RGBD+mask+act history +-wandb_run_id: '1000_0' # all ++# wandb_run_id: '1000_0' # all ++ ++# all books, 10/20 demos per book ++# RGBD+mask+act history ++wandb_run_id: '1007_0' ++ + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +@@ -174,7 +179,7 @@ residual_weight: '${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name + + # old_experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${name_of_expert_demo}_valgroups_${val_num_groups}/${old_feature_type}/${seed}/${agent.name}/${wandb_run_id} + experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +-final_experiment_dir: ${experiment_dir}/${oc.generate_run_id} ++final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} + # ${now:%H%M%S} + hydra: + run: +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 5ed3eb4..21e335c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -148,7 +148,14 @@ remote_host = 'grasp-login1' + # ] + + run_id_list = [ +- '1000_0', ++ # '1002_0', ++ # '1009_1', ++ # '1009_2', ++ # '1022_0', ++ # '1022_0', ++ # '1023_0', ++ # '1007_0', ++ '1007_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 3482538..531c122 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -463,6 +463,8 @@ class Workspace: + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + ++ table_data = [table_data] ++ + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) +@@ -471,11 +473,13 @@ class Workspace: + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ +- "jobType": "eval", ++ "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, +- "state": "finished" ++ "state": "finished", ++ "config.grasped_obj_name": grasped_obj_name, ++ "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -503,6 +507,8 @@ class Workspace: + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub ++ ++ total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]} \ No newline at end of file diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f44e1c42e00ae465b2429b9fc796c6ca41b590cd --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[3, 5, 0.6, 0.12214982517170042, 0.9567054509203167, 0.2083774209605767, 0.9455957635345953, 0.25968933110274955, 0.8756684585654417, 0.2662367970218884, 0.8798631577924128]]} \ No newline at end of file diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3a5caae9d785a885dc8fafbc3b1bc8dc899a6ba --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e739aa3a1064d7a115c648ec192bb1b5b34388ac6d31936a7ef6fd1ca52ae4ea +size 678402 diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c40124fc43aa9234a041f0c5b2f73630fee63d3 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a45cc58516bfa8280677b9bd2af88e662726e4efdf25b8e31cbbf385486eed +size 594378 diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68ac988a741e39e6fbb27b273d0a97a784b5d644 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1c0520eab44850309611a44e67873cd28908f9005d185df5b339ad1874bcd7 +size 1157154 diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd6a24d715fd9d7fcb2bd344eadeef7ba9074b5f --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e47918912079386856849116782c06f1a9ea624d9c21c42d468e6a4df4f115 +size 1157639 diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68b9df691aebf9b8fa67c459c85760cacd28d802 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43d65e2ea74e852f815b6bdd0071163c9bd16ea1f6cadc977d529abe360e33d +size 1171802 diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9f74d2332eae9e5c22fa3b1bcc712a897b951844 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log @@ -0,0 +1,5075 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735697009.760999]: resetting environment +[INFO] [1735697009.769098]: cleared current plan +[INFO] [1735697009.770207]: moving to home +[INFO] [1735697010.772656]: reached home +[INFO] [1735697010.773928]: reset action history +[INFO] [1735697013.485471]: environment reset +Starting episode 0 +[INFO] [1735697013.487878]: resetting environment +[INFO] [1735697013.490855]: cleared current plan +[INFO] [1735697013.492037]: moving to home +[INFO] [1735697014.494966]: reached home +[INFO] [1735697014.496640]: reset action history +[INFO] [1735697017.206495]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735697024.976218]: resetting environment +[INFO] [1735697024.983484]: cleared current plan +[INFO] [1735697024.985295]: moving to home +[INFO] [1735697025.987622]: reached home +[INFO] [1735697025.988843]: reset action history +[INFO] [1735697028.696227]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/episode_rosbags/episode_0_2024-12-31-21-03-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1740046.log +started roslaunch server http://158.130.50.37:41355/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1740345] +process[print_text-2]: started with pid [1740346] +process[rosbag_record-3]: started with pid [1740347] +started bagging! +[INFO] [1735697029.479267]: received mask_obs with header seq: 666 +stamp: + secs: 1735697029 + nsecs: 482137918 +frame_id: '' and grasped_object_mask header seq: 129992 +stamp: + secs: 1735697029 + nsecs: 257171154 +frame_id: "camera_color_optical_frame" +[INFO] [1735697029.664356]: received mask_obs with header seq: 667 +stamp: + secs: 1735697029 + nsecs: 667510747 +frame_id: '' and grasped_object_mask header seq: 129998 +stamp: + secs: 1735697029 + nsecs: 457011223 +frame_id: "camera_color_optical_frame" +[INFO] [1735697030.061790]: received mask_obs with header seq: 668 +stamp: + secs: 1735697030 + nsecs: 65122127 +frame_id: '' and grasped_object_mask header seq: 130010 +stamp: + secs: 1735697029 + nsecs: 856688976 +frame_id: "camera_color_optical_frame" +[INFO] [1735697030.252742]: received mask_obs with header seq: 669 +stamp: + secs: 1735697030 + nsecs: 255935907 +frame_id: '' and grasped_object_mask header seq: 130015 +stamp: + secs: 1735697030 + nsecs: 23221731 +frame_id: "camera_color_optical_frame" +[INFO] [1735697030.432002]: received mask_obs with header seq: 670 +stamp: + secs: 1735697030 + nsecs: 435265064 +frame_id: '' and grasped_object_mask header seq: 130021 +stamp: + secs: 1735697030 + nsecs: 223060131 +frame_id: "camera_color_optical_frame" +[INFO] [1735697030.609348]: received mask_obs with header seq: 671 +stamp: + secs: 1735697030 + nsecs: 612568378 +frame_id: '' and grasped_object_mask header seq: 130026 +stamp: + secs: 1735697030 + nsecs: 389591694 +frame_id: "camera_color_optical_frame" +[INFO] [1735697030.810292]: received mask_obs with header seq: 672 +stamp: + secs: 1735697030 + nsecs: 813551425 +frame_id: '' and grasped_object_mask header seq: 130032 +stamp: + secs: 1735697030 + nsecs: 589431286 +frame_id: "camera_color_optical_frame" +[INFO] [1735697031.015201]: received mask_obs with header seq: 673 +stamp: + secs: 1735697031 + nsecs: 18344640 +frame_id: '' and grasped_object_mask header seq: 130038 +stamp: + secs: 1735697030 + nsecs: 789269924 +frame_id: "camera_color_optical_frame" +[INFO] [1735697031.213204]: received mask_obs with header seq: 674 +stamp: + secs: 1735697031 + nsecs: 216224431 +frame_id: '' and grasped_object_mask header seq: 130044 +stamp: + secs: 1735697030 + nsecs: 989105463 +frame_id: "camera_color_optical_frame" +[INFO] [1735697031.416957]: received mask_obs with header seq: 675 +stamp: + secs: 1735697031 + nsecs: 420257806 +frame_id: '' and grasped_object_mask header seq: 130050 +stamp: + secs: 1735697031 + nsecs: 188938856 +frame_id: "camera_color_optical_frame" +[INFO] [1735697031.625431]: received mask_obs with header seq: 676 +stamp: + secs: 1735697031 + nsecs: 628485679 +frame_id: '' and grasped_object_mask header seq: 130056 +stamp: + secs: 1735697031 + nsecs: 388772249 +frame_id: "camera_color_optical_frame" +[INFO] [1735697031.816841]: received mask_obs with header seq: 677 +stamp: + secs: 1735697031 + nsecs: 820145845 +frame_id: '' and grasped_object_mask header seq: 130062 +stamp: + secs: 1735697031 + nsecs: 588605165 +frame_id: "camera_color_optical_frame" +[INFO] [1735697032.015981]: received mask_obs with header seq: 678 +stamp: + secs: 1735697032 + nsecs: 18965005 +frame_id: '' and grasped_object_mask header seq: 130068 +stamp: + secs: 1735697031 + nsecs: 788438320 +frame_id: "camera_color_optical_frame" +[INFO] [1735697032.219372]: received mask_obs with header seq: 679 +stamp: + secs: 1735697032 + nsecs: 222604036 +frame_id: '' and grasped_object_mask header seq: 130074 +stamp: + secs: 1735697031 + nsecs: 988274097 +frame_id: "camera_color_optical_frame" +[INFO] [1735697032.416682]: received mask_obs with header seq: 680 +stamp: + secs: 1735697032 + nsecs: 420006513 +frame_id: '' and grasped_object_mask header seq: 130080 +stamp: + secs: 1735697032 + nsecs: 188110352 +frame_id: "camera_color_optical_frame" +[INFO] [1735697032.626621]: received mask_obs with header seq: 681 +stamp: + secs: 1735697032 + nsecs: 629800796 +frame_id: '' and grasped_object_mask header seq: 130086 +stamp: + secs: 1735697032 + nsecs: 387946367 +frame_id: "camera_color_optical_frame" +[INFO] [1735697032.836651]: received mask_obs with header seq: 682 +stamp: + secs: 1735697032 + nsecs: 839714765 +frame_id: '' and grasped_object_mask header seq: 130092 +stamp: + secs: 1735697032 + nsecs: 587782145 +frame_id: "camera_color_optical_frame" +[INFO] [1735697033.047703]: received mask_obs with header seq: 683 +stamp: + secs: 1735697033 + nsecs: 50909996 +frame_id: '' and grasped_object_mask header seq: 130098 +stamp: + secs: 1735697032 + nsecs: 787618160 +frame_id: "camera_color_optical_frame" +[INFO] [1735697033.247066]: received mask_obs with header seq: 684 +stamp: + secs: 1735697033 + nsecs: 250217676 +frame_id: '' and grasped_object_mask header seq: 130105 +stamp: + secs: 1735697033 + nsecs: 21990299 +frame_id: "camera_color_optical_frame" +[INFO] [1735697033.445256]: received mask_obs with header seq: 685 +stamp: + secs: 1735697033 + nsecs: 448351144 +frame_id: '' and grasped_object_mask header seq: 130111 +stamp: + secs: 1735697033 + nsecs: 224182129 +frame_id: "camera_color_optical_frame" +[INFO] [1735697033.646401]: received mask_obs with header seq: 686 +stamp: + secs: 1735697033 + nsecs: 649179220 +frame_id: '' and grasped_object_mask header seq: 130117 +stamp: + secs: 1735697033 + nsecs: 426461220 +frame_id: "camera_color_optical_frame" +[INFO] [1735697033.836156]: received mask_obs with header seq: 687 +stamp: + secs: 1735697033 + nsecs: 837815761 +frame_id: '' and grasped_object_mask header seq: 130123 +stamp: + secs: 1735697033 + nsecs: 628827572 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.026786]: received mask_obs with header seq: 688 +stamp: + secs: 1735697034 + nsecs: 30062437 +frame_id: '' and grasped_object_mask header seq: 130128 +stamp: + secs: 1735697033 + nsecs: 797533512 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.222770]: received mask_obs with header seq: 689 +stamp: + secs: 1735697034 + nsecs: 225755691 +frame_id: '' and grasped_object_mask header seq: 130134 +stamp: + secs: 1735697033 + nsecs: 998984337 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.418375]: received mask_obs with header seq: 690 +stamp: + secs: 1735697034 + nsecs: 421696424 +frame_id: '' and grasped_object_mask header seq: 130140 +stamp: + secs: 1735697034 + nsecs: 198607206 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.617167]: received mask_obs with header seq: 691 +stamp: + secs: 1735697034 + nsecs: 620347023 +frame_id: '' and grasped_object_mask header seq: 130146 +stamp: + secs: 1735697034 + nsecs: 398220062 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.811974]: received mask_obs with header seq: 692 +stamp: + secs: 1735697034 + nsecs: 815235614 +frame_id: '' and grasped_object_mask header seq: 130152 +stamp: + secs: 1735697034 + nsecs: 597820520 +frame_id: "camera_color_optical_frame" +[INFO] [1735697034.991357]: received mask_obs with header seq: 693 +stamp: + secs: 1735697034 + nsecs: 994648933 +frame_id: '' and grasped_object_mask header seq: 130157 +stamp: + secs: 1735697034 + nsecs: 764145374 +frame_id: "camera_color_optical_frame" +[INFO] [1735697035.184263]: received mask_obs with header seq: 694 +stamp: + secs: 1735697035 + nsecs: 187585353 +frame_id: '' and grasped_object_mask header seq: 130163 +stamp: + secs: 1735697034 + nsecs: 963734388 +frame_id: "camera_color_optical_frame" +[INFO] [1735697035.382029]: received mask_obs with header seq: 695 +stamp: + secs: 1735697035 + nsecs: 385329008 +frame_id: '' and grasped_object_mask header seq: 130169 +stamp: + secs: 1735697035 + nsecs: 163343430 +frame_id: "camera_color_optical_frame" +[INFO] [1735697035.578844]: received mask_obs with header seq: 696 +stamp: + secs: 1735697035 + nsecs: 581949949 +frame_id: '' and grasped_object_mask header seq: 130175 +stamp: + secs: 1735697035 + nsecs: 362941027 +frame_id: "camera_color_optical_frame" +[INFO] [1735697035.777183]: received mask_obs with header seq: 697 +stamp: + secs: 1735697035 + nsecs: 780269384 +frame_id: '' and grasped_object_mask header seq: 130181 +stamp: + secs: 1735697035 + nsecs: 562526464 +frame_id: "camera_color_optical_frame" +[INFO] [1735697035.978301]: received mask_obs with header seq: 698 +stamp: + secs: 1735697035 + nsecs: 981595277 +frame_id: '' and grasped_object_mask header seq: 130187 +stamp: + secs: 1735697035 + nsecs: 762099981 +frame_id: "camera_color_optical_frame" +[INFO] [1735697036.172335]: received mask_obs with header seq: 699 +stamp: + secs: 1735697036 + nsecs: 174495935 +frame_id: '' and grasped_object_mask header seq: 130192 +stamp: + secs: 1735697035 + nsecs: 928406715 +frame_id: "camera_color_optical_frame" +[INFO] [1735697036.378860]: received mask_obs with header seq: 700 +stamp: + secs: 1735697036 + nsecs: 382054090 +frame_id: '' and grasped_object_mask header seq: 130198 +stamp: + secs: 1735697036 + nsecs: 128015041 +frame_id: "camera_color_optical_frame" +[INFO] [1735697036.579381]: received mask_obs with header seq: 701 +stamp: + secs: 1735697036 + nsecs: 582427978 +frame_id: '' and grasped_object_mask header seq: 130205 +stamp: + secs: 1735697036 + nsecs: 360877275 +frame_id: "camera_color_optical_frame" +[INFO] [1735697036.776819]: received mask_obs with header seq: 702 +stamp: + secs: 1735697036 + nsecs: 779738426 +frame_id: '' and grasped_object_mask header seq: 130211 +stamp: + secs: 1735697036 + nsecs: 560459614 +frame_id: "camera_color_optical_frame" +[INFO] [1735697036.973475]: received mask_obs with header seq: 703 +stamp: + secs: 1735697036 + nsecs: 976621627 +frame_id: '' and grasped_object_mask header seq: 130217 +stamp: + secs: 1735697036 + nsecs: 760030270 +frame_id: "camera_color_optical_frame" +[INFO] [1735697037.167278]: received mask_obs with header seq: 704 +stamp: + secs: 1735697037 + nsecs: 170339107 +frame_id: '' and grasped_object_mask header seq: 130223 +stamp: + secs: 1735697036 + nsecs: 959599495 +frame_id: "camera_color_optical_frame" +[INFO] [1735697037.357730]: received mask_obs with header seq: 705 +stamp: + secs: 1735697037 + nsecs: 360550642 +frame_id: '' and grasped_object_mask header seq: 130228 +stamp: + secs: 1735697037 + nsecs: 125942945 +frame_id: "camera_color_optical_frame" +[INFO] [1735697037.555962]: received mask_obs with header seq: 706 +stamp: + secs: 1735697037 + nsecs: 558752775 +frame_id: '' and grasped_object_mask header seq: 130234 +stamp: + secs: 1735697037 + nsecs: 325543642 +frame_id: "camera_color_optical_frame" +[INFO] [1735697037.750129]: received mask_obs with header seq: 707 +stamp: + secs: 1735697037 + nsecs: 753348350 +frame_id: '' and grasped_object_mask header seq: 130240 +stamp: + secs: 1735697037 + nsecs: 525132895 +frame_id: "camera_color_optical_frame" +[INFO] [1735697037.945930]: received mask_obs with header seq: 708 +stamp: + secs: 1735697037 + nsecs: 949216842 +frame_id: '' and grasped_object_mask header seq: 130246 +stamp: + secs: 1735697037 + nsecs: 724710226 +frame_id: "camera_color_optical_frame" +[INFO] [1735697038.144703]: received mask_obs with header seq: 709 +stamp: + secs: 1735697038 + nsecs: 147819042 +frame_id: '' and grasped_object_mask header seq: 130252 +stamp: + secs: 1735697037 + nsecs: 924275875 +frame_id: "camera_color_optical_frame" +[INFO] [1735697038.343864]: received mask_obs with header seq: 710 +stamp: + secs: 1735697038 + nsecs: 347180366 +frame_id: '' and grasped_object_mask header seq: 130258 +stamp: + secs: 1735697038 + nsecs: 123881578 +frame_id: "camera_color_optical_frame" +[INFO] [1735697038.540511]: received mask_obs with header seq: 711 +stamp: + secs: 1735697038 + nsecs: 542929649 +frame_id: '' and grasped_object_mask header seq: 130264 +stamp: + secs: 1735697038 + nsecs: 323474407 +frame_id: "camera_color_optical_frame" +[INFO] [1735697038.738646]: received mask_obs with header seq: 712 +stamp: + secs: 1735697038 + nsecs: 741070985 +frame_id: '' and grasped_object_mask header seq: 130270 +stamp: + secs: 1735697038 + nsecs: 523055077 +frame_id: "camera_color_optical_frame" +[INFO] [1735697038.942774]: received mask_obs with header seq: 713 +stamp: + secs: 1735697038 + nsecs: 945895910 +frame_id: '' and grasped_object_mask header seq: 130276 +stamp: + secs: 1735697038 + nsecs: 722624063 +frame_id: "camera_color_optical_frame" +[INFO] [1735697039.139810]: received mask_obs with header seq: 714 +stamp: + secs: 1735697039 + nsecs: 142163991 +frame_id: '' and grasped_object_mask header seq: 130282 +stamp: + secs: 1735697038 + nsecs: 922180891 +frame_id: "camera_color_optical_frame" +[INFO] [1735697039.329188]: received mask_obs with header seq: 715 +stamp: + secs: 1735697039 + nsecs: 332473039 +frame_id: '' and grasped_object_mask header seq: 130287 +stamp: + secs: 1735697039 + nsecs: 88521481 +frame_id: "camera_color_optical_frame" +[INFO] [1735697039.524110]: received mask_obs with header seq: 716 +stamp: + secs: 1735697039 + nsecs: 527199983 +frame_id: '' and grasped_object_mask header seq: 130293 +stamp: + secs: 1735697039 + nsecs: 288118601 +frame_id: "camera_color_optical_frame" +[INFO] [1735697039.717768]: received mask_obs with header seq: 717 +stamp: + secs: 1735697039 + nsecs: 720966339 +frame_id: '' and grasped_object_mask header seq: 130299 +stamp: + secs: 1735697039 + nsecs: 487704039 +frame_id: "camera_color_optical_frame" +[INFO] [1735697039.913279]: received mask_obs with header seq: 718 +stamp: + secs: 1735697039 + nsecs: 916568040 +frame_id: '' and grasped_object_mask header seq: 130305 +stamp: + secs: 1735697039 + nsecs: 687277794 +frame_id: "camera_color_optical_frame" +[INFO] [1735697040.113314]: received mask_obs with header seq: 719 +stamp: + secs: 1735697040 + nsecs: 116538047 +frame_id: '' and grasped_object_mask header seq: 130311 +stamp: + secs: 1735697039 + nsecs: 886840343 +frame_id: "camera_color_optical_frame" +[INFO] [1735697040.316506]: received mask_obs with header seq: 720 +stamp: + secs: 1735697040 + nsecs: 319647550 +frame_id: '' and grasped_object_mask header seq: 130317 +stamp: + secs: 1735697040 + nsecs: 86444378 +frame_id: "camera_color_optical_frame" +[INFO] [1735697040.506905]: received mask_obs with header seq: 721 +stamp: + secs: 1735697040 + nsecs: 510005712 +frame_id: '' and grasped_object_mask header seq: 130323 +stamp: + secs: 1735697040 + nsecs: 286047697 +frame_id: "camera_color_optical_frame" +[INFO] [1735697040.710692]: received mask_obs with header seq: 722 +stamp: + secs: 1735697040 + nsecs: 714020490 +frame_id: '' and grasped_object_mask header seq: 130329 +stamp: + secs: 1735697040 + nsecs: 485639095 +frame_id: "camera_color_optical_frame" +[INFO] [1735697040.909067]: received mask_obs with header seq: 723 +stamp: + secs: 1735697040 + nsecs: 912368297 +frame_id: '' and grasped_object_mask header seq: 130335 +stamp: + secs: 1735697040 + nsecs: 685219049 +frame_id: "camera_color_optical_frame" +[INFO] [1735697041.109329]: received mask_obs with header seq: 724 +stamp: + secs: 1735697041 + nsecs: 112538099 +frame_id: '' and grasped_object_mask header seq: 130341 +stamp: + secs: 1735697040 + nsecs: 884787321 +frame_id: "camera_color_optical_frame" +[INFO] [1735697041.308212]: received mask_obs with header seq: 725 +stamp: + secs: 1735697041 + nsecs: 311323404 +frame_id: '' and grasped_object_mask header seq: 130347 +stamp: + secs: 1735697041 + nsecs: 84392309 +frame_id: "camera_color_optical_frame" +[INFO] [1735697041.507876]: received mask_obs with header seq: 726 +stamp: + secs: 1735697041 + nsecs: 510500431 +frame_id: '' and grasped_object_mask header seq: 130353 +stamp: + secs: 1735697041 + nsecs: 283993959 +frame_id: "camera_color_optical_frame" +[INFO] [1735697041.707174]: received mask_obs with header seq: 727 +stamp: + secs: 1735697041 + nsecs: 710418462 +frame_id: '' and grasped_object_mask header seq: 130359 +stamp: + secs: 1735697041 + nsecs: 483583689 +frame_id: "camera_color_optical_frame" +[INFO] [1735697041.907289]: received mask_obs with header seq: 728 +stamp: + secs: 1735697041 + nsecs: 909138679 +frame_id: '' and grasped_object_mask header seq: 130365 +stamp: + secs: 1735697041 + nsecs: 683161974 +frame_id: "camera_color_optical_frame" +[INFO] [1735697042.104273]: received mask_obs with header seq: 729 +stamp: + secs: 1735697042 + nsecs: 107597351 +frame_id: '' and grasped_object_mask header seq: 130371 +stamp: + secs: 1735697041 + nsecs: 882728100 +frame_id: "camera_color_optical_frame" +[INFO] [1735697042.295417]: received mask_obs with header seq: 730 +stamp: + secs: 1735697042 + nsecs: 297474622 +frame_id: '' and grasped_object_mask header seq: 130376 +stamp: + secs: 1735697042 + nsecs: 49069166 +frame_id: "camera_color_optical_frame" +[INFO] [1735697042.488494]: received mask_obs with header seq: 731 +stamp: + secs: 1735697042 + nsecs: 491780757 +frame_id: '' and grasped_object_mask header seq: 130382 +stamp: + secs: 1735697042 + nsecs: 248680353 +frame_id: "camera_color_optical_frame" +[INFO] [1735697042.681605]: received mask_obs with header seq: 732 +stamp: + secs: 1735697042 + nsecs: 684845924 +frame_id: '' and grasped_object_mask header seq: 130388 +stamp: + secs: 1735697042 + nsecs: 448280334 +frame_id: "camera_color_optical_frame" +[INFO] [1735697042.880849]: received mask_obs with header seq: 733 +stamp: + secs: 1735697042 + nsecs: 883918523 +frame_id: '' and grasped_object_mask header seq: 130394 +stamp: + secs: 1735697042 + nsecs: 647868633 +frame_id: "camera_color_optical_frame" +[INFO] [1735697043.077265]: received mask_obs with header seq: 734 +stamp: + secs: 1735697043 + nsecs: 80549478 +frame_id: '' and grasped_object_mask header seq: 130400 +stamp: + secs: 1735697042 + nsecs: 847445488 +frame_id: "camera_color_optical_frame" +[INFO] [1735697043.276997]: received mask_obs with header seq: 735 +stamp: + secs: 1735697043 + nsecs: 280098199 +frame_id: '' and grasped_object_mask header seq: 130406 +stamp: + secs: 1735697043 + nsecs: 47046900 +frame_id: "camera_color_optical_frame" +[INFO] [1735697043.478906]: received mask_obs with header seq: 736 +stamp: + secs: 1735697043 + nsecs: 481942415 +frame_id: '' and grasped_object_mask header seq: 130412 +stamp: + secs: 1735697043 + nsecs: 246653080 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1735697043.539864]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735697043.678166]: received mask_obs with header seq: 737 +stamp: + secs: 1735697043 + nsecs: 680661916 +frame_id: '' and grasped_object_mask header seq: 130418 +stamp: + secs: 1735697043 + nsecs: 446247578 +frame_id: "camera_color_optical_frame" +[INFO] [1735697044.318460]: Stored episode 1. +[INFO] [1735697044.319454]: Saving video... +[INFO] [1735697044.547930]: Video saved! +Starting episode 1 +[INFO] [1735697044.560751]: resetting environment +[INFO] [1735697044.572121]: cleared current plan +[INFO] [1735697044.572575]: moving to home +[INFO] [1735697047.774218]: reached home +[INFO] [1735697047.774678]: reset action history +[INFO] [1735697050.486906]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735697076.195790]: resetting environment +[INFO] [1735697076.198340]: cleared current plan +[INFO] [1735697076.198558]: moving to home +[INFO] [1735697077.200248]: reached home +[INFO] [1735697077.200667]: reset action history +[INFO] [1735697079.906574]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/episode_rosbags/episode_1_2024-12-31-21-04-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1740046.log +started roslaunch server http://158.130.50.37:43073/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1740476] +process[print_text-5]: started with pid [1740477] +process[rosbag_record-6]: started with pid [1740478] +started bagging! +[INFO] [1735697080.581485]: received mask_obs with header seq: 738 +stamp: + secs: 1735697080 + nsecs: 584553241 +frame_id: '' and grasped_object_mask header seq: 131527 +stamp: + secs: 1735697080 + nsecs: 382540703 +frame_id: "camera_color_optical_frame" +[INFO] [1735697080.784828]: received mask_obs with header seq: 739 +stamp: + secs: 1735697080 + nsecs: 787961959 +frame_id: '' and grasped_object_mask header seq: 131533 +stamp: + secs: 1735697080 + nsecs: 582379580 +frame_id: "camera_color_optical_frame" +[INFO] [1735697080.986295]: received mask_obs with header seq: 740 +stamp: + secs: 1735697080 + nsecs: 989354610 +frame_id: '' and grasped_object_mask header seq: 131539 +stamp: + secs: 1735697080 + nsecs: 782217503 +frame_id: "camera_color_optical_frame" +[INFO] [1735697081.189442]: received mask_obs with header seq: 741 +stamp: + secs: 1735697081 + nsecs: 192714452 +frame_id: '' and grasped_object_mask header seq: 131545 +stamp: + secs: 1735697080 + nsecs: 982055187 +frame_id: "camera_color_optical_frame" +[INFO] [1735697081.397651]: received mask_obs with header seq: 742 +stamp: + secs: 1735697081 + nsecs: 399443864 +frame_id: '' and grasped_object_mask header seq: 131551 +stamp: + secs: 1735697081 + nsecs: 181884050 +frame_id: "camera_color_optical_frame" +[INFO] [1735697081.601730]: received mask_obs with header seq: 743 +stamp: + secs: 1735697081 + nsecs: 605080127 +frame_id: '' and grasped_object_mask header seq: 131557 +stamp: + secs: 1735697081 + nsecs: 381710529 +frame_id: "camera_color_optical_frame" +[INFO] [1735697081.804707]: received mask_obs with header seq: 744 +stamp: + secs: 1735697081 + nsecs: 807755470 +frame_id: '' and grasped_object_mask header seq: 131563 +stamp: + secs: 1735697081 + nsecs: 581536055 +frame_id: "camera_color_optical_frame" +[INFO] [1735697082.008811]: received mask_obs with header seq: 745 +stamp: + secs: 1735697082 + nsecs: 11657238 +frame_id: '' and grasped_object_mask header seq: 131569 +stamp: + secs: 1735697081 + nsecs: 781362295 +frame_id: "camera_color_optical_frame" +[INFO] [1735697082.214597]: received mask_obs with header seq: 746 +stamp: + secs: 1735697082 + nsecs: 217945337 +frame_id: '' and grasped_object_mask header seq: 131575 +stamp: + secs: 1735697081 + nsecs: 981187105 +frame_id: "camera_color_optical_frame" +[INFO] [1735697082.414478]: received mask_obs with header seq: 747 +stamp: + secs: 1735697082 + nsecs: 417708396 +frame_id: '' and grasped_object_mask header seq: 131581 +stamp: + secs: 1735697082 + nsecs: 181025028 +frame_id: "camera_color_optical_frame" +[INFO] [1735697082.613076]: received mask_obs with header seq: 748 +stamp: + secs: 1735697082 + nsecs: 616410017 +frame_id: '' and grasped_object_mask header seq: 131587 +stamp: + secs: 1735697082 + nsecs: 380866051 +frame_id: "camera_color_optical_frame" +[INFO] [1735697082.806067]: received mask_obs with header seq: 749 +stamp: + secs: 1735697082 + nsecs: 808989524 +frame_id: '' and grasped_object_mask header seq: 131593 +stamp: + secs: 1735697082 + nsecs: 580707073 +frame_id: "camera_color_optical_frame" +[INFO] [1735697083.007405]: received mask_obs with header seq: 750 +stamp: + secs: 1735697083 + nsecs: 8825302 +frame_id: '' and grasped_object_mask header seq: 131599 +stamp: + secs: 1735697082 + nsecs: 780548096 +frame_id: "camera_color_optical_frame" +[INFO] [1735697083.206602]: received mask_obs with header seq: 751 +stamp: + secs: 1735697083 + nsecs: 209768295 +frame_id: '' and grasped_object_mask header seq: 131605 +stamp: + secs: 1735697082 + nsecs: 980389118 +frame_id: "camera_color_optical_frame" +[INFO] [1735697083.410747]: received mask_obs with header seq: 752 +stamp: + secs: 1735697083 + nsecs: 414041996 +frame_id: '' and grasped_object_mask header seq: 131611 +stamp: + secs: 1735697083 + nsecs: 180230618 +frame_id: "camera_color_optical_frame" +[INFO] [1735697083.608772]: received mask_obs with header seq: 753 +stamp: + secs: 1735697083 + nsecs: 612146615 +frame_id: '' and grasped_object_mask header seq: 131617 +stamp: + secs: 1735697083 + nsecs: 380071163 +frame_id: "camera_color_optical_frame" +[INFO] [1735697083.805584]: received mask_obs with header seq: 754 +stamp: + secs: 1735697083 + nsecs: 808498620 +frame_id: '' and grasped_object_mask header seq: 131623 +stamp: + secs: 1735697083 + nsecs: 579911947 +frame_id: "camera_color_optical_frame" +[INFO] [1735697084.007289]: received mask_obs with header seq: 755 +stamp: + secs: 1735697084 + nsecs: 10365247 +frame_id: '' and grasped_object_mask header seq: 131629 +stamp: + secs: 1735697083 + nsecs: 779752493 +frame_id: "camera_color_optical_frame" +[INFO] [1735697084.206714]: received mask_obs with header seq: 756 +stamp: + secs: 1735697084 + nsecs: 210100889 +frame_id: '' and grasped_object_mask header seq: 131635 +stamp: + secs: 1735697083 + nsecs: 979593515 +frame_id: "camera_color_optical_frame" +[INFO] [1735697084.407330]: received mask_obs with header seq: 757 +stamp: + secs: 1735697084 + nsecs: 410625934 +frame_id: '' and grasped_object_mask header seq: 131641 +stamp: + secs: 1735697084 + nsecs: 179425240 +frame_id: "camera_color_optical_frame" +[INFO] [1735697084.607605]: received mask_obs with header seq: 758 +stamp: + secs: 1735697084 + nsecs: 610873699 +frame_id: '' and grasped_object_mask header seq: 131647 +stamp: + secs: 1735697084 + nsecs: 379255772 +frame_id: "camera_color_optical_frame" +[INFO] [1735697084.807022]: received mask_obs with header seq: 759 +stamp: + secs: 1735697084 + nsecs: 809793233 +frame_id: '' and grasped_object_mask header seq: 131653 +stamp: + secs: 1735697084 + nsecs: 579085112 +frame_id: "camera_color_optical_frame" +[INFO] [1735697085.010021]: received mask_obs with header seq: 760 +stamp: + secs: 1735697085 + nsecs: 13384103 +frame_id: '' and grasped_object_mask header seq: 131659 +stamp: + secs: 1735697084 + nsecs: 778913975 +frame_id: "camera_color_optical_frame" +[INFO] [1735697085.218436]: received mask_obs with header seq: 761 +stamp: + secs: 1735697085 + nsecs: 221604347 +frame_id: '' and grasped_object_mask header seq: 131665 +stamp: + secs: 1735697084 + nsecs: 978742599 +frame_id: "camera_color_optical_frame" +[INFO] [1735697085.433679]: received mask_obs with header seq: 762 +stamp: + secs: 1735697085 + nsecs: 437028646 +frame_id: '' and grasped_object_mask header seq: 131671 +stamp: + secs: 1735697085 + nsecs: 179877758 +frame_id: "camera_color_optical_frame" +[INFO] [1735697085.640038]: received mask_obs with header seq: 763 +stamp: + secs: 1735697085 + nsecs: 643064498 +frame_id: '' and grasped_object_mask header seq: 131678 +stamp: + secs: 1735697085 + nsecs: 415791750 +frame_id: "camera_color_optical_frame" +[INFO] [1735697085.842082]: received mask_obs with header seq: 764 +stamp: + secs: 1735697085 + nsecs: 845478296 +frame_id: '' and grasped_object_mask header seq: 131684 +stamp: + secs: 1735697085 + nsecs: 618098736 +frame_id: "camera_color_optical_frame" +[INFO] [1735697086.046579]: received mask_obs with header seq: 765 +stamp: + secs: 1735697086 + nsecs: 49854755 +frame_id: '' and grasped_object_mask header seq: 131690 +stamp: + secs: 1735697085 + nsecs: 820493698 +frame_id: "camera_color_optical_frame" +[INFO] [1735697086.244968]: received mask_obs with header seq: 766 +stamp: + secs: 1735697086 + nsecs: 248279571 +frame_id: '' and grasped_object_mask header seq: 131696 +stamp: + secs: 1735697086 + nsecs: 22976398 +frame_id: "camera_color_optical_frame" +[INFO] [1735697086.440694]: received mask_obs with header seq: 767 +stamp: + secs: 1735697086 + nsecs: 443669319 +frame_id: '' and grasped_object_mask header seq: 131702 +stamp: + secs: 1735697086 + nsecs: 223384380 +frame_id: "camera_color_optical_frame" +[INFO] [1735697086.639929]: received mask_obs with header seq: 768 +stamp: + secs: 1735697086 + nsecs: 642789602 +frame_id: '' and grasped_object_mask header seq: 131708 +stamp: + secs: 1735697086 + nsecs: 422991037 +frame_id: "camera_color_optical_frame" +[INFO] [1735697086.834410]: received mask_obs with header seq: 769 +stamp: + secs: 1735697086 + nsecs: 837554216 +frame_id: '' and grasped_object_mask header seq: 131714 +stamp: + secs: 1735697086 + nsecs: 622586250 +frame_id: "camera_color_optical_frame" +[INFO] [1735697087.034322]: received mask_obs with header seq: 770 +stamp: + secs: 1735697087 + nsecs: 37662029 +frame_id: '' and grasped_object_mask header seq: 131720 +stamp: + secs: 1735697086 + nsecs: 822169781 +frame_id: "camera_color_optical_frame" +[INFO] [1735697087.237373]: received mask_obs with header seq: 771 +stamp: + secs: 1735697087 + nsecs: 240727663 +frame_id: '' and grasped_object_mask header seq: 131726 +stamp: + secs: 1735697087 + nsecs: 21741390 +frame_id: "camera_color_optical_frame" +[INFO] [1735697087.446348]: received mask_obs with header seq: 772 +stamp: + secs: 1735697087 + nsecs: 449371576 +frame_id: '' and grasped_object_mask header seq: 131732 +stamp: + secs: 1735697087 + nsecs: 221352100 +frame_id: "camera_color_optical_frame" +[INFO] [1735697087.650631]: received mask_obs with header seq: 773 +stamp: + secs: 1735697087 + nsecs: 653758525 +frame_id: '' and grasped_object_mask header seq: 131738 +stamp: + secs: 1735697087 + nsecs: 420949936 +frame_id: "camera_color_optical_frame" +[INFO] [1735697087.844734]: received mask_obs with header seq: 774 +stamp: + secs: 1735697087 + nsecs: 847890615 +frame_id: '' and grasped_object_mask header seq: 131744 +stamp: + secs: 1735697087 + nsecs: 620534658 +frame_id: "camera_color_optical_frame" +[INFO] [1735697088.040885]: received mask_obs with header seq: 775 +stamp: + secs: 1735697088 + nsecs: 44032812 +frame_id: '' and grasped_object_mask header seq: 131750 +stamp: + secs: 1735697087 + nsecs: 820107460 +frame_id: "camera_color_optical_frame" +[INFO] [1735697088.238174]: received mask_obs with header seq: 776 +stamp: + secs: 1735697088 + nsecs: 241184711 +frame_id: '' and grasped_object_mask header seq: 131756 +stamp: + secs: 1735697088 + nsecs: 19668102 +frame_id: "camera_color_optical_frame" +[INFO] [1735697088.436255]: received mask_obs with header seq: 777 +stamp: + secs: 1735697088 + nsecs: 439369201 +frame_id: '' and grasped_object_mask header seq: 131762 +stamp: + secs: 1735697088 + nsecs: 219256163 +frame_id: "camera_color_optical_frame" +[INFO] [1735697088.636140]: received mask_obs with header seq: 778 +stamp: + secs: 1735697088 + nsecs: 639476537 +frame_id: '' and grasped_object_mask header seq: 131768 +stamp: + secs: 1735697088 + nsecs: 418851852 +frame_id: "camera_color_optical_frame" +[INFO] [1735697088.832448]: received mask_obs with header seq: 779 +stamp: + secs: 1735697088 + nsecs: 835810661 +frame_id: '' and grasped_object_mask header seq: 131774 +stamp: + secs: 1735697088 + nsecs: 618436337 +frame_id: "camera_color_optical_frame" +[INFO] [1735697089.029604]: received mask_obs with header seq: 780 +stamp: + secs: 1735697089 + nsecs: 32856941 +frame_id: '' and grasped_object_mask header seq: 131780 +stamp: + secs: 1735697088 + nsecs: 818007708 +frame_id: "camera_color_optical_frame" +[INFO] [1735697089.217200]: received mask_obs with header seq: 781 +stamp: + secs: 1735697089 + nsecs: 220467567 +frame_id: '' and grasped_object_mask header seq: 131785 +stamp: + secs: 1735697088 + nsecs: 984307528 +frame_id: "camera_color_optical_frame" +[INFO] [1735697089.414392]: received mask_obs with header seq: 782 +stamp: + secs: 1735697089 + nsecs: 417703866 +frame_id: '' and grasped_object_mask header seq: 131791 +stamp: + secs: 1735697089 + nsecs: 183892488 +frame_id: "camera_color_optical_frame" +[INFO] [1735697089.613438]: received mask_obs with header seq: 783 +stamp: + secs: 1735697089 + nsecs: 616755008 +frame_id: '' and grasped_object_mask header seq: 131797 +stamp: + secs: 1735697089 + nsecs: 383498907 +frame_id: "camera_color_optical_frame" +[INFO] [1735697089.806355]: received mask_obs with header seq: 784 +stamp: + secs: 1735697089 + nsecs: 809401750 +frame_id: '' and grasped_object_mask header seq: 131803 +stamp: + secs: 1735697089 + nsecs: 583094120 +frame_id: "camera_color_optical_frame" +[INFO] [1735697090.006622]: received mask_obs with header seq: 785 +stamp: + secs: 1735697090 + nsecs: 9886980 +frame_id: '' and grasped_object_mask header seq: 131809 +stamp: + secs: 1735697089 + nsecs: 782677412 +frame_id: "camera_color_optical_frame" +[INFO] [1735697090.206521]: received mask_obs with header seq: 786 +stamp: + secs: 1735697090 + nsecs: 208995103 +frame_id: '' and grasped_object_mask header seq: 131815 +stamp: + secs: 1735697089 + nsecs: 982249260 +frame_id: "camera_color_optical_frame" +[INFO] [1735697090.404763]: received mask_obs with header seq: 787 +stamp: + secs: 1735697090 + nsecs: 408142328 +frame_id: '' and grasped_object_mask header seq: 131821 +stamp: + secs: 1735697090 + nsecs: 181841612 +frame_id: "camera_color_optical_frame" +[INFO] [1735697090.607126]: received mask_obs with header seq: 788 +stamp: + secs: 1735697090 + nsecs: 609677553 +frame_id: '' and grasped_object_mask header seq: 131827 +stamp: + secs: 1735697090 + nsecs: 381454706 +frame_id: "camera_color_optical_frame" +[INFO] [1735697090.806404]: received mask_obs with header seq: 789 +stamp: + secs: 1735697090 + nsecs: 809685230 +frame_id: '' and grasped_object_mask header seq: 131833 +stamp: + secs: 1735697090 + nsecs: 581057072 +frame_id: "camera_color_optical_frame" +[INFO] [1735697091.006605]: received mask_obs with header seq: 790 +stamp: + secs: 1735697091 + nsecs: 9515285 +frame_id: '' and grasped_object_mask header seq: 131839 +stamp: + secs: 1735697090 + nsecs: 780647516 +frame_id: "camera_color_optical_frame" +[INFO] [1735697091.205919]: received mask_obs with header seq: 791 +stamp: + secs: 1735697091 + nsecs: 208737134 +frame_id: '' and grasped_object_mask header seq: 131845 +stamp: + secs: 1735697090 + nsecs: 980226278 +frame_id: "camera_color_optical_frame" +[INFO] [1735697091.405243]: received mask_obs with header seq: 792 +stamp: + secs: 1735697091 + nsecs: 408586263 +frame_id: '' and grasped_object_mask header seq: 131851 +stamp: + secs: 1735697091 + nsecs: 179820538 +frame_id: "camera_color_optical_frame" +[INFO] [1735697091.609761]: received mask_obs with header seq: 793 +stamp: + secs: 1735697091 + nsecs: 613085031 +frame_id: '' and grasped_object_mask header seq: 131857 +stamp: + secs: 1735697091 + nsecs: 379429817 +frame_id: "camera_color_optical_frame" +[INFO] [1735697091.804231]: received mask_obs with header seq: 794 +stamp: + secs: 1735697091 + nsecs: 807398557 +frame_id: '' and grasped_object_mask header seq: 131863 +stamp: + secs: 1735697091 + nsecs: 579027653 +frame_id: "camera_color_optical_frame" +[INFO] [1735697092.006438]: received mask_obs with header seq: 795 +stamp: + secs: 1735697092 + nsecs: 9773731 +frame_id: '' and grasped_object_mask header seq: 131869 +stamp: + secs: 1735697091 + nsecs: 778614998 +frame_id: "camera_color_optical_frame" +[INFO] [1735697092.204756]: received mask_obs with header seq: 796 +stamp: + secs: 1735697092 + nsecs: 208005905 +frame_id: '' and grasped_object_mask header seq: 131875 +stamp: + secs: 1735697091 + nsecs: 978189707 +frame_id: "camera_color_optical_frame" +[INFO] [1735697092.407643]: received mask_obs with header seq: 797 +stamp: + secs: 1735697092 + nsecs: 410707473 +frame_id: '' and grasped_object_mask header seq: 131881 +stamp: + secs: 1735697092 + nsecs: 177780867 +frame_id: "camera_color_optical_frame" +[INFO] [1735697092.608110]: received mask_obs with header seq: 798 +stamp: + secs: 1735697092 + nsecs: 611157417 +frame_id: '' and grasped_object_mask header seq: 131887 +stamp: + secs: 1735697092 + nsecs: 377388239 +frame_id: "camera_color_optical_frame" +[INFO] [1735697092.806712]: received mask_obs with header seq: 799 +stamp: + secs: 1735697092 + nsecs: 809417009 +frame_id: '' and grasped_object_mask header seq: 131893 +stamp: + secs: 1735697092 + nsecs: 576984167 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1735697092.978484]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735697093.007784]: received mask_obs with header seq: 800 +stamp: + secs: 1735697093 + nsecs: 10927200 +frame_id: '' and grasped_object_mask header seq: 131899 +stamp: + secs: 1735697092 + nsecs: 776568174 +frame_id: "camera_color_optical_frame" +[INFO] [1735697093.705468]: Stored episode 2. +[INFO] [1735697093.705686]: Saving video... +[WARN] [1735697094.218693]: Plan exhausted +[WARN] [1735697094.245992]: Plan exhausted +[WARN] [1735697094.285416]: Plan exhausted +[WARN] [1735697094.326714]: Plan exhausted +[INFO] [1735697094.292059]: Video saved! +Starting episode 2 +[INFO] [1735697094.334606]: resetting environment +[INFO] [1735697094.353237]: cleared current plan +[INFO] [1735697094.354209]: moving to home +[INFO] [1735697097.856758]: reached home +[INFO] [1735697097.857832]: reset action history +[INFO] [1735697100.626990]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735697119.212584]: resetting environment +[INFO] [1735697119.220237]: cleared current plan +[INFO] [1735697119.221608]: moving to home +[INFO] [1735697120.227311]: reached home +[INFO] [1735697120.232435]: reset action history +[INFO] [1735697122.942076]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/episode_rosbags/episode_2_2024-12-31-21-05-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1740046.log +started roslaunch server http://158.130.50.37:36193/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1740629] +process[print_text-8]: started with pid [1740630] +process[rosbag_record-9]: started with pid [1740631] +started bagging! +[INFO] [1735697123.623672]: received mask_obs with header seq: 801 +stamp: + secs: 1735697123 + nsecs: 626425027 +frame_id: '' and grasped_object_mask header seq: 132819 +stamp: + secs: 1735697123 + nsecs: 415324211 +frame_id: "camera_color_optical_frame" +[INFO] [1735697123.815997]: received mask_obs with header seq: 802 +stamp: + secs: 1735697123 + nsecs: 819065570 +frame_id: '' and grasped_object_mask header seq: 132825 +stamp: + secs: 1735697123 + nsecs: 614914894 +frame_id: "camera_color_optical_frame" +[INFO] [1735697124.021259]: received mask_obs with header seq: 803 +stamp: + secs: 1735697124 + nsecs: 24495601 +frame_id: '' and grasped_object_mask header seq: 132831 +stamp: + secs: 1735697123 + nsecs: 814494610 +frame_id: "camera_color_optical_frame" +[INFO] [1735697124.221899]: received mask_obs with header seq: 804 +stamp: + secs: 1735697124 + nsecs: 225194692 +frame_id: '' and grasped_object_mask header seq: 132837 +stamp: + secs: 1735697124 + nsecs: 14061213 +frame_id: "camera_color_optical_frame" +[INFO] [1735697124.426671]: received mask_obs with header seq: 805 +stamp: + secs: 1735697124 + nsecs: 430097818 +frame_id: '' and grasped_object_mask header seq: 132843 +stamp: + secs: 1735697124 + nsecs: 213624001 +frame_id: "camera_color_optical_frame" +[INFO] [1735697124.631482]: received mask_obs with header seq: 806 +stamp: + secs: 1735697124 + nsecs: 634105443 +frame_id: '' and grasped_object_mask header seq: 132849 +stamp: + secs: 1735697124 + nsecs: 413230419 +frame_id: "camera_color_optical_frame" +[INFO] [1735697124.836200]: received mask_obs with header seq: 807 +stamp: + secs: 1735697124 + nsecs: 839601039 +frame_id: '' and grasped_object_mask header seq: 132855 +stamp: + secs: 1735697124 + nsecs: 612824917 +frame_id: "camera_color_optical_frame" +[INFO] [1735697125.041492]: received mask_obs with header seq: 808 +stamp: + secs: 1735697125 + nsecs: 44452905 +frame_id: '' and grasped_object_mask header seq: 132861 +stamp: + secs: 1735697124 + nsecs: 812407732 +frame_id: "camera_color_optical_frame" +[INFO] [1735697125.239625]: received mask_obs with header seq: 809 +stamp: + secs: 1735697125 + nsecs: 243024110 +frame_id: '' and grasped_object_mask header seq: 132867 +stamp: + secs: 1735697125 + nsecs: 11979580 +frame_id: "camera_color_optical_frame" +[INFO] [1735697125.441821]: received mask_obs with header seq: 810 +stamp: + secs: 1735697125 + nsecs: 444758415 +frame_id: '' and grasped_object_mask header seq: 132873 +stamp: + secs: 1735697125 + nsecs: 211547375 +frame_id: "camera_color_optical_frame" +[INFO] [1735697125.646399]: received mask_obs with header seq: 811 +stamp: + secs: 1735697125 + nsecs: 649295330 +frame_id: '' and grasped_object_mask header seq: 132879 +stamp: + secs: 1735697125 + nsecs: 411165237 +frame_id: "camera_color_optical_frame" +[INFO] [1735697125.846970]: received mask_obs with header seq: 812 +stamp: + secs: 1735697125 + nsecs: 850408554 +frame_id: '' and grasped_object_mask header seq: 132885 +stamp: + secs: 1735697125 + nsecs: 610771656 +frame_id: "camera_color_optical_frame" +[INFO] [1735697126.046716]: received mask_obs with header seq: 813 +stamp: + secs: 1735697126 + nsecs: 49710035 +frame_id: '' and grasped_object_mask header seq: 132891 +stamp: + secs: 1735697125 + nsecs: 810366869 +frame_id: "camera_color_optical_frame" +[INFO] [1735697126.241106]: received mask_obs with header seq: 814 +stamp: + secs: 1735697126 + nsecs: 244502544 +frame_id: '' and grasped_object_mask header seq: 132897 +stamp: + secs: 1735697126 + nsecs: 9950399 +frame_id: "camera_color_optical_frame" +[INFO] [1735697126.448655]: received mask_obs with header seq: 815 +stamp: + secs: 1735697126 + nsecs: 451832771 +frame_id: '' and grasped_object_mask header seq: 132903 +stamp: + secs: 1735697126 + nsecs: 209530354 +frame_id: "camera_color_optical_frame" +[INFO] [1735697126.644853]: received mask_obs with header seq: 816 +stamp: + secs: 1735697126 + nsecs: 647961139 +frame_id: '' and grasped_object_mask header seq: 132909 +stamp: + secs: 1735697126 + nsecs: 409143209 +frame_id: "camera_color_optical_frame" +[INFO] [1735697126.850225]: received mask_obs with header seq: 817 +stamp: + secs: 1735697126 + nsecs: 853058099 +frame_id: '' and grasped_object_mask header seq: 132915 +stamp: + secs: 1735697126 + nsecs: 608744860 +frame_id: "camera_color_optical_frame" +[INFO] [1735697127.046796]: received mask_obs with header seq: 818 +stamp: + secs: 1735697127 + nsecs: 49820661 +frame_id: '' and grasped_object_mask header seq: 132921 +stamp: + secs: 1735697126 + nsecs: 808334827 +frame_id: "camera_color_optical_frame" +[INFO] [1735697127.246557]: received mask_obs with header seq: 819 +stamp: + secs: 1735697127 + nsecs: 249085187 +frame_id: '' and grasped_object_mask header seq: 132927 +stamp: + secs: 1735697127 + nsecs: 7912874 +frame_id: "camera_color_optical_frame" +[INFO] [1735697127.441046]: received mask_obs with header seq: 820 +stamp: + secs: 1735697127 + nsecs: 444103002 +frame_id: '' and grasped_object_mask header seq: 132933 +stamp: + secs: 1735697127 + nsecs: 207486391 +frame_id: "camera_color_optical_frame" +[INFO] [1735697127.641613]: received mask_obs with header seq: 821 +stamp: + secs: 1735697127 + nsecs: 644908905 +frame_id: '' and grasped_object_mask header seq: 132939 +stamp: + secs: 1735697127 + nsecs: 407100201 +frame_id: "camera_color_optical_frame" +[INFO] [1735697127.846789]: received mask_obs with header seq: 822 +stamp: + secs: 1735697127 + nsecs: 849693536 +frame_id: '' and grasped_object_mask header seq: 132945 +stamp: + secs: 1735697127 + nsecs: 606701136 +frame_id: "camera_color_optical_frame" +[INFO] [1735697128.040151]: received mask_obs with header seq: 823 +stamp: + secs: 1735697128 + nsecs: 43149232 +frame_id: '' and grasped_object_mask header seq: 132951 +stamp: + secs: 1735697127 + nsecs: 806290627 +frame_id: "camera_color_optical_frame" +[INFO] [1735697128.239180]: received mask_obs with header seq: 824 +stamp: + secs: 1735697128 + nsecs: 242568969 +frame_id: '' and grasped_object_mask header seq: 132957 +stamp: + secs: 1735697128 + nsecs: 5868196 +frame_id: "camera_color_optical_frame" +[INFO] [1735697128.440948]: received mask_obs with header seq: 825 +stamp: + secs: 1735697128 + nsecs: 444337368 +frame_id: '' and grasped_object_mask header seq: 132963 +stamp: + secs: 1735697128 + nsecs: 205440998 +frame_id: "camera_color_optical_frame" +[INFO] [1735697128.646266]: received mask_obs with header seq: 826 +stamp: + secs: 1735697128 + nsecs: 648753404 +frame_id: '' and grasped_object_mask header seq: 132969 +stamp: + secs: 1735697128 + nsecs: 405052900 +frame_id: "camera_color_optical_frame" +[INFO] [1735697128.841186]: received mask_obs with header seq: 827 +stamp: + secs: 1735697128 + nsecs: 844595193 +frame_id: '' and grasped_object_mask header seq: 132975 +stamp: + secs: 1735697128 + nsecs: 604653358 +frame_id: "camera_color_optical_frame" +[INFO] [1735697129.041577]: received mask_obs with header seq: 828 +stamp: + secs: 1735697129 + nsecs: 44616699 +frame_id: '' and grasped_object_mask header seq: 132981 +stamp: + secs: 1735697128 + nsecs: 804242849 +frame_id: "camera_color_optical_frame" +[INFO] [1735697129.239693]: received mask_obs with header seq: 829 +stamp: + secs: 1735697129 + nsecs: 242666244 +frame_id: '' and grasped_object_mask header seq: 132987 +stamp: + secs: 1735697129 + nsecs: 3819942 +frame_id: "camera_color_optical_frame" +[INFO] [1735697129.439011]: received mask_obs with header seq: 830 +stamp: + secs: 1735697129 + nsecs: 442422866 +frame_id: '' and grasped_object_mask header seq: 132993 +stamp: + secs: 1735697129 + nsecs: 203535080 +frame_id: "camera_color_optical_frame" +[INFO] [1735697129.644114]: received mask_obs with header seq: 831 +stamp: + secs: 1735697129 + nsecs: 647521972 +frame_id: '' and grasped_object_mask header seq: 132999 +stamp: + secs: 1735697129 + nsecs: 404453278 +frame_id: "camera_color_optical_frame" +[INFO] [1735697129.840012]: received mask_obs with header seq: 832 +stamp: + secs: 1735697129 + nsecs: 843252420 +frame_id: '' and grasped_object_mask header seq: 133005 +stamp: + secs: 1735697129 + nsecs: 605459690 +frame_id: "camera_color_optical_frame" +[INFO] [1735697130.042912]: received mask_obs with header seq: 833 +stamp: + secs: 1735697130 + nsecs: 46109437 +frame_id: '' and grasped_object_mask header seq: 133011 +stamp: + secs: 1735697129 + nsecs: 806554556 +frame_id: "camera_color_optical_frame" +[INFO] [1735697130.238511]: received mask_obs with header seq: 834 +stamp: + secs: 1735697130 + nsecs: 241795063 +frame_id: '' and grasped_object_mask header seq: 133017 +stamp: + secs: 1735697130 + nsecs: 7738352 +frame_id: "camera_color_optical_frame" +[INFO] [1735697130.439252]: received mask_obs with header seq: 835 +stamp: + secs: 1735697130 + nsecs: 442636489 +frame_id: '' and grasped_object_mask header seq: 133023 +stamp: + secs: 1735697130 + nsecs: 208845377 +frame_id: "camera_color_optical_frame" +[INFO] [1735697130.642556]: received mask_obs with header seq: 836 +stamp: + secs: 1735697130 + nsecs: 645940303 +frame_id: '' and grasped_object_mask header seq: 133029 +stamp: + secs: 1735697130 + nsecs: 408694506 +frame_id: "camera_color_optical_frame" +[INFO] [1735697130.839711]: received mask_obs with header seq: 837 +stamp: + secs: 1735697130 + nsecs: 842904090 +frame_id: '' and grasped_object_mask header seq: 133035 +stamp: + secs: 1735697130 + nsecs: 608544111 +frame_id: "camera_color_optical_frame" +[INFO] [1735697131.039897]: received mask_obs with header seq: 838 +stamp: + secs: 1735697131 + nsecs: 43297767 +frame_id: '' and grasped_object_mask header seq: 133041 +stamp: + secs: 1735697130 + nsecs: 808394432 +frame_id: "camera_color_optical_frame" +[INFO] [1735697131.240411]: received mask_obs with header seq: 839 +stamp: + secs: 1735697131 + nsecs: 243764162 +frame_id: '' and grasped_object_mask header seq: 133047 +stamp: + secs: 1735697131 + nsecs: 8245468 +frame_id: "camera_color_optical_frame" +[INFO] [1735697131.439970]: received mask_obs with header seq: 840 +stamp: + secs: 1735697131 + nsecs: 443295240 +frame_id: '' and grasped_object_mask header seq: 133053 +stamp: + secs: 1735697131 + nsecs: 208096266 +frame_id: "camera_color_optical_frame" +[INFO] [1735697131.642713]: received mask_obs with header seq: 841 +stamp: + secs: 1735697131 + nsecs: 646033525 +frame_id: '' and grasped_object_mask header seq: 133059 +stamp: + secs: 1735697131 + nsecs: 407932758 +frame_id: "camera_color_optical_frame" +[INFO] [1735697131.839473]: received mask_obs with header seq: 842 +stamp: + secs: 1735697131 + nsecs: 842846393 +frame_id: '' and grasped_object_mask header seq: 133065 +stamp: + secs: 1735697131 + nsecs: 607769251 +frame_id: "camera_color_optical_frame" +[INFO] [1735697132.038108]: received mask_obs with header seq: 843 +stamp: + secs: 1735697132 + nsecs: 41489601 +frame_id: '' and grasped_object_mask header seq: 133071 +stamp: + secs: 1735697131 + nsecs: 807605982 +frame_id: "camera_color_optical_frame" +[INFO] [1735697132.241042]: received mask_obs with header seq: 844 +stamp: + secs: 1735697132 + nsecs: 244441032 +frame_id: '' and grasped_object_mask header seq: 133077 +stamp: + secs: 1735697132 + nsecs: 7442713 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 112720014 for nearest: 1735697132319999219 - target: 1735697132207279205 at idx 128 +[INFO] [1735697132.447779]: received mask_obs with header seq: 845 +stamp: + secs: 1735697132 + nsecs: 451058626 +frame_id: '' and grasped_object_mask header seq: 133083 +stamp: + secs: 1735697132 + nsecs: 207279205 +frame_id: "camera_color_optical_frame" +[INFO] [1735697132.645190]: received mask_obs with header seq: 846 +stamp: + secs: 1735697132 + nsecs: 648563385 +frame_id: '' and grasped_object_mask header seq: 133089 +stamp: + secs: 1735697132 + nsecs: 407120466 +frame_id: "camera_color_optical_frame" +[INFO] [1735697132.841486]: received mask_obs with header seq: 847 +stamp: + secs: 1735697132 + nsecs: 844819307 +frame_id: '' and grasped_object_mask header seq: 133095 +stamp: + secs: 1735697132 + nsecs: 606960535 +frame_id: "camera_color_optical_frame" +[INFO] [1735697133.041931]: received mask_obs with header seq: 848 +stamp: + secs: 1735697133 + nsecs: 45271396 +frame_id: '' and grasped_object_mask header seq: 133101 +stamp: + secs: 1735697132 + nsecs: 806800604 +frame_id: "camera_color_optical_frame" +[INFO] [1735697133.241120]: received mask_obs with header seq: 849 +stamp: + secs: 1735697133 + nsecs: 243759632 +frame_id: '' and grasped_object_mask header seq: 133107 +stamp: + secs: 1735697133 + nsecs: 6640911 +frame_id: "camera_color_optical_frame" +[INFO] [1735697133.439397]: received mask_obs with header seq: 850 +stamp: + secs: 1735697133 + nsecs: 442402839 +frame_id: '' and grasped_object_mask header seq: 133113 +stamp: + secs: 1735697133 + nsecs: 206481695 +frame_id: "camera_color_optical_frame" +[INFO] [1735697133.641516]: received mask_obs with header seq: 851 +stamp: + secs: 1735697133 + nsecs: 644722700 +frame_id: '' and grasped_object_mask header seq: 133119 +stamp: + secs: 1735697133 + nsecs: 406327724 +frame_id: "camera_color_optical_frame" +[INFO] [1735697133.848971]: received mask_obs with header seq: 852 +stamp: + secs: 1735697133 + nsecs: 852335453 +frame_id: '' and grasped_object_mask header seq: 133125 +stamp: + secs: 1735697133 + nsecs: 606174946 +frame_id: "camera_color_optical_frame" +[INFO] [1735697134.044640]: received mask_obs with header seq: 853 +stamp: + secs: 1735697134 + nsecs: 47961235 +frame_id: '' and grasped_object_mask header seq: 133131 +stamp: + secs: 1735697133 + nsecs: 806021690 +frame_id: "camera_color_optical_frame" +[INFO] [1735697134.247901]: received mask_obs with header seq: 854 +stamp: + secs: 1735697134 + nsecs: 251021623 +frame_id: '' and grasped_object_mask header seq: 133137 +stamp: + secs: 1735697134 + nsecs: 5868912 +frame_id: "camera_color_optical_frame" +[INFO] [1735697134.440454]: received mask_obs with header seq: 855 +stamp: + secs: 1735697134 + nsecs: 443806409 +frame_id: '' and grasped_object_mask header seq: 133143 +stamp: + secs: 1735697134 + nsecs: 205716133 +frame_id: "camera_color_optical_frame" +[INFO] [1735697134.643718]: received mask_obs with header seq: 856 +stamp: + secs: 1735697134 + nsecs: 647100210 +frame_id: '' and grasped_object_mask header seq: 133149 +stamp: + secs: 1735697134 + nsecs: 405558586 +frame_id: "camera_color_optical_frame" +[INFO] [1735697134.839668]: received mask_obs with header seq: 857 +stamp: + secs: 1735697134 + nsecs: 843054056 +frame_id: '' and grasped_object_mask header seq: 133155 +stamp: + secs: 1735697134 + nsecs: 605401516 +frame_id: "camera_color_optical_frame" +[INFO] [1735697135.047790]: received mask_obs with header seq: 858 +stamp: + secs: 1735697135 + nsecs: 51108837 +frame_id: '' and grasped_object_mask header seq: 133161 +stamp: + secs: 1735697134 + nsecs: 805245876 +frame_id: "camera_color_optical_frame" +[INFO] [1735697135.251849]: received mask_obs with header seq: 859 +stamp: + secs: 1735697135 + nsecs: 255061149 +frame_id: '' and grasped_object_mask header seq: 133167 +stamp: + secs: 1735697135 + nsecs: 5089283 +frame_id: "camera_color_optical_frame" +[INFO] [1735697135.452442]: received mask_obs with header seq: 860 +stamp: + secs: 1735697135 + nsecs: 455800294 +frame_id: '' and grasped_object_mask header seq: 133173 +stamp: + secs: 1735697135 + nsecs: 204932690 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 115235512 for nearest: 1735697135520006602 - target: 1735697135404771090 at idx 169 +[INFO] [1735697135.665479]: received mask_obs with header seq: 861 +stamp: + secs: 1735697135 + nsecs: 668282032 +frame_id: '' and grasped_object_mask header seq: 133179 +stamp: + secs: 1735697135 + nsecs: 404771090 +frame_id: "camera_color_optical_frame" +[INFO] [1735697135.871511]: received mask_obs with header seq: 862 +stamp: + secs: 1735697135 + nsecs: 874832868 +frame_id: '' and grasped_object_mask header seq: 133186 +stamp: + secs: 1735697135 + nsecs: 637916565 +frame_id: "camera_color_optical_frame" +[INFO] [1735697136.072145]: received mask_obs with header seq: 863 +stamp: + secs: 1735697136 + nsecs: 75319766 +frame_id: '' and grasped_object_mask header seq: 133192 +stamp: + secs: 1735697135 + nsecs: 837754726 +frame_id: "camera_color_optical_frame" +[INFO] [1735697136.273118]: received mask_obs with header seq: 864 +stamp: + secs: 1735697136 + nsecs: 276509761 +frame_id: '' and grasped_object_mask header seq: 133198 +stamp: + secs: 1735697136 + nsecs: 37592888 +frame_id: "camera_color_optical_frame" +[INFO] [1735697136.481591]: received mask_obs with header seq: 865 +stamp: + secs: 1735697136 + nsecs: 485010385 +frame_id: '' and grasped_object_mask header seq: 133204 +stamp: + secs: 1735697136 + nsecs: 237430811 +frame_id: "camera_color_optical_frame" +[INFO] [1735697136.680552]: received mask_obs with header seq: 866 +stamp: + secs: 1735697136 + nsecs: 683830499 +frame_id: '' and grasped_object_mask header seq: 133210 +stamp: + secs: 1735697136 + nsecs: 437268734 +frame_id: "camera_color_optical_frame" +[INFO] [1735697136.876856]: received mask_obs with header seq: 867 +stamp: + secs: 1735697136 + nsecs: 880066633 +frame_id: '' and grasped_object_mask header seq: 133216 +stamp: + secs: 1735697136 + nsecs: 637106419 +frame_id: "camera_color_optical_frame" +[INFO] [1735697137.077503]: received mask_obs with header seq: 868 +stamp: + secs: 1735697137 + nsecs: 80914258 +frame_id: '' and grasped_object_mask header seq: 133222 +stamp: + secs: 1735697136 + nsecs: 836945057 +frame_id: "camera_color_optical_frame" +[INFO] [1735697137.271670]: received mask_obs with header seq: 869 +stamp: + secs: 1735697137 + nsecs: 274961233 +frame_id: '' and grasped_object_mask header seq: 133228 +stamp: + secs: 1735697137 + nsecs: 36782503 +frame_id: "camera_color_optical_frame" +[INFO] [1735697137.468521]: received mask_obs with header seq: 870 +stamp: + secs: 1735697137 + nsecs: 471842527 +frame_id: '' and grasped_object_mask header seq: 133234 +stamp: + secs: 1735697137 + nsecs: 236619949 +frame_id: "camera_color_optical_frame" +[INFO] [1735697137.669285]: received mask_obs with header seq: 871 +stamp: + secs: 1735697137 + nsecs: 672680139 +frame_id: '' and grasped_object_mask header seq: 133240 +stamp: + secs: 1735697137 + nsecs: 436456680 +frame_id: "camera_color_optical_frame" +[INFO] [1735697137.867800]: received mask_obs with header seq: 872 +stamp: + secs: 1735697137 + nsecs: 871068477 +frame_id: '' and grasped_object_mask header seq: 133246 +stamp: + secs: 1735697137 + nsecs: 636293888 +frame_id: "camera_color_optical_frame" +[INFO] [1735697138.066692]: received mask_obs with header seq: 873 +stamp: + secs: 1735697138 + nsecs: 69917201 +frame_id: '' and grasped_object_mask header seq: 133252 +stamp: + secs: 1735697137 + nsecs: 836131096 +frame_id: "camera_color_optical_frame" +[INFO] [1735697138.264787]: received mask_obs with header seq: 874 +stamp: + secs: 1735697138 + nsecs: 268116712 +frame_id: '' and grasped_object_mask header seq: 133257 +stamp: + secs: 1735697138 + nsecs: 2662420 +frame_id: "camera_color_optical_frame" +[INFO] [1735697138.458515]: received mask_obs with header seq: 875 +stamp: + secs: 1735697138 + nsecs: 461943149 +frame_id: '' and grasped_object_mask header seq: 133264 +stamp: + secs: 1735697138 + nsecs: 235806465 +frame_id: "camera_color_optical_frame" +[INFO] [1735697138.664642]: received mask_obs with header seq: 876 +stamp: + secs: 1735697138 + nsecs: 668043613 +frame_id: '' and grasped_object_mask header seq: 133269 +stamp: + secs: 1735697138 + nsecs: 402338505 +frame_id: "camera_color_optical_frame" +[INFO] [1735697138.864768]: received mask_obs with header seq: 877 +stamp: + secs: 1735697138 + nsecs: 867805957 +frame_id: '' and grasped_object_mask header seq: 133276 +stamp: + secs: 1735697138 + nsecs: 635484934 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 118025000 for nearest: 1735697138920043166 - target: 1735697138802018166 at idx 216 +[INFO] [1735697139.052597]: received mask_obs with header seq: 878 +stamp: + secs: 1735697139 + nsecs: 55504560 +frame_id: '' and grasped_object_mask header seq: 133281 +stamp: + secs: 1735697138 + nsecs: 802018166 +frame_id: "camera_color_optical_frame" +[INFO] [1735697139.247849]: received mask_obs with header seq: 879 +stamp: + secs: 1735697139 + nsecs: 251286983 +frame_id: '' and grasped_object_mask header seq: 133287 +stamp: + secs: 1735697139 + nsecs: 1857519 +frame_id: "camera_color_optical_frame" +[INFO] [1735697139.446632]: received mask_obs with header seq: 880 +stamp: + secs: 1735697139 + nsecs: 449092864 +frame_id: '' and grasped_object_mask header seq: 133293 +stamp: + secs: 1735697139 + nsecs: 201696873 +frame_id: "camera_color_optical_frame" +[INFO] [1735697139.643098]: received mask_obs with header seq: 881 +stamp: + secs: 1735697139 + nsecs: 646493911 +frame_id: '' and grasped_object_mask header seq: 133299 +stamp: + secs: 1735697139 + nsecs: 401527166 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 118686003 for nearest: 1735697139720042748 - target: 1735697139601356745 at idx 226 +[INFO] [1735697139.847820]: received mask_obs with header seq: 882 +stamp: + secs: 1735697139 + nsecs: 851273536 +frame_id: '' and grasped_object_mask header seq: 133305 +stamp: + secs: 1735697139 + nsecs: 601356745 +frame_id: "camera_color_optical_frame" +[INFO] [1735697140.046751]: received mask_obs with header seq: 883 +stamp: + secs: 1735697140 + nsecs: 49975156 +frame_id: '' and grasped_object_mask header seq: 133311 +stamp: + secs: 1735697139 + nsecs: 801187038 +frame_id: "camera_color_optical_frame" +[INFO] [1735697140.240505]: received mask_obs with header seq: 884 +stamp: + secs: 1735697140 + nsecs: 243931293 +frame_id: '' and grasped_object_mask header seq: 133317 +stamp: + secs: 1735697140 + nsecs: 1015663 +frame_id: "camera_color_optical_frame" +[INFO] [1735697140.437836]: received mask_obs with header seq: 885 +stamp: + secs: 1735697140 + nsecs: 441156387 +frame_id: '' and grasped_object_mask header seq: 133323 +stamp: + secs: 1735697140 + nsecs: 200844049 +frame_id: "camera_color_optical_frame" +[INFO] [1735697140.641660]: received mask_obs with header seq: 886 +stamp: + secs: 1735697140 + nsecs: 645082473 +frame_id: '' and grasped_object_mask header seq: 133329 +stamp: + secs: 1735697140 + nsecs: 400677681 +frame_id: "camera_color_optical_frame" +[INFO] [1735697140.840166]: received mask_obs with header seq: 887 +stamp: + secs: 1735697140 + nsecs: 843372583 +frame_id: '' and grasped_object_mask header seq: 133335 +stamp: + secs: 1735697140 + nsecs: 600511312 +frame_id: "camera_color_optical_frame" +[INFO] [1735697141.039121]: received mask_obs with header seq: 888 +stamp: + secs: 1735697141 + nsecs: 42544603 +frame_id: '' and grasped_object_mask header seq: 133341 +stamp: + secs: 1735697140 + nsecs: 800344467 +frame_id: "camera_color_optical_frame" +[INFO] [1735697141.241042]: received mask_obs with header seq: 889 +stamp: + secs: 1735697141 + nsecs: 244458436 +frame_id: '' and grasped_object_mask header seq: 133347 +stamp: + secs: 1735697141 + nsecs: 178576 +frame_id: "camera_color_optical_frame" +[INFO] [1735697141.442601]: received mask_obs with header seq: 890 +stamp: + secs: 1735697141 + nsecs: 445880651 +frame_id: '' and grasped_object_mask header seq: 133353 +stamp: + secs: 1735697141 + nsecs: 200011015 +frame_id: "camera_color_optical_frame" +[INFO] [1735697141.642873]: received mask_obs with header seq: 891 +stamp: + secs: 1735697141 + nsecs: 646043062 +frame_id: '' and grasped_object_mask header seq: 133359 +stamp: + secs: 1735697141 + nsecs: 399852753 +frame_id: "camera_color_optical_frame" +[INFO] [1735697141.838106]: received mask_obs with header seq: 892 +stamp: + secs: 1735697141 + nsecs: 841254472 +frame_id: '' and grasped_object_mask header seq: 133365 +stamp: + secs: 1735697141 + nsecs: 599694490 +frame_id: "camera_color_optical_frame" +[INFO] [1735697142.041163]: received mask_obs with header seq: 893 +stamp: + secs: 1735697142 + nsecs: 44402599 +frame_id: '' and grasped_object_mask header seq: 133371 +stamp: + secs: 1735697141 + nsecs: 799536705 +frame_id: "camera_color_optical_frame" +[INFO] [1735697142.243593]: received mask_obs with header seq: 894 +stamp: + secs: 1735697142 + nsecs: 247004985 +frame_id: '' and grasped_object_mask header seq: 133377 +stamp: + secs: 1735697141 + nsecs: 999378920 +frame_id: "camera_color_optical_frame" +[INFO] [1735697142.442499]: received mask_obs with header seq: 895 +stamp: + secs: 1735697142 + nsecs: 445721626 +frame_id: '' and grasped_object_mask header seq: 133383 +stamp: + secs: 1735697142 + nsecs: 199222088 +frame_id: "camera_color_optical_frame" +[INFO] [1735697142.642424]: received mask_obs with header seq: 896 +stamp: + secs: 1735697142 + nsecs: 645538806 +frame_id: '' and grasped_object_mask header seq: 133389 +stamp: + secs: 1735697142 + nsecs: 399059296 +frame_id: "camera_color_optical_frame" +[INFO] [1735697142.841803]: received mask_obs with header seq: 897 +stamp: + secs: 1735697142 + nsecs: 845240116 +frame_id: '' and grasped_object_mask header seq: 133395 +stamp: + secs: 1735697142 + nsecs: 598896503 +frame_id: "camera_color_optical_frame" +[INFO] [1735697143.042720]: received mask_obs with header seq: 898 +stamp: + secs: 1735697143 + nsecs: 46126842 +frame_id: '' and grasped_object_mask header seq: 133401 +stamp: + secs: 1735697142 + nsecs: 798733711 +frame_id: "camera_color_optical_frame" +[INFO] [1735697143.242392]: received mask_obs with header seq: 899 +stamp: + secs: 1735697143 + nsecs: 245726823 +frame_id: '' and grasped_object_mask header seq: 133407 +stamp: + secs: 1735697142 + nsecs: 998570442 +frame_id: "camera_color_optical_frame" +[INFO] [1735697143.443728]: received mask_obs with header seq: 900 +stamp: + secs: 1735697143 + nsecs: 447125196 +frame_id: '' and grasped_object_mask header seq: 133413 +stamp: + secs: 1735697143 + nsecs: 198407412 +frame_id: "camera_color_optical_frame" +[INFO] [1735697143.647997]: received mask_obs with header seq: 901 +stamp: + secs: 1735697143 + nsecs: 648491621 +frame_id: '' and grasped_object_mask header seq: 133419 +stamp: + secs: 1735697143 + nsecs: 398245573 +frame_id: "camera_color_optical_frame" +[INFO] [1735697143.843749]: received mask_obs with header seq: 902 +stamp: + secs: 1735697143 + nsecs: 846910715 +frame_id: '' and grasped_object_mask header seq: 133425 +stamp: + secs: 1735697143 + nsecs: 598084450 +frame_id: "camera_color_optical_frame" +[INFO] [1735697144.045419]: received mask_obs with header seq: 903 +stamp: + secs: 1735697144 + nsecs: 48370122 +frame_id: '' and grasped_object_mask header seq: 133431 +stamp: + secs: 1735697143 + nsecs: 797922611 +frame_id: "camera_color_optical_frame" +[INFO] [1735697144.255849]: received mask_obs with header seq: 904 +stamp: + secs: 1735697144 + nsecs: 259036064 +frame_id: '' and grasped_object_mask header seq: 133437 +stamp: + secs: 1735697143 + nsecs: 997760534 +frame_id: "camera_color_optical_frame" +[INFO] [1735697144.452568]: received mask_obs with header seq: 905 +stamp: + secs: 1735697144 + nsecs: 455792427 +frame_id: '' and grasped_object_mask header seq: 133444 +stamp: + secs: 1735697144 + nsecs: 230905294 +frame_id: "camera_color_optical_frame" +[INFO] [1735697144.649919]: received mask_obs with header seq: 906 +stamp: + secs: 1735697144 + nsecs: 653359413 +frame_id: '' and grasped_object_mask header seq: 133450 +stamp: + secs: 1735697144 + nsecs: 430742741 +frame_id: "camera_color_optical_frame" +[INFO] [1735697144.843138]: received mask_obs with header seq: 907 +stamp: + secs: 1735697144 + nsecs: 846471309 +frame_id: '' and grasped_object_mask header seq: 133455 +stamp: + secs: 1735697144 + nsecs: 597273588 +frame_id: "camera_color_optical_frame" +[INFO] [1735697145.043605]: received mask_obs with header seq: 908 +stamp: + secs: 1735697145 + nsecs: 46904325 +frame_id: '' and grasped_object_mask header seq: 133462 +stamp: + secs: 1735697144 + nsecs: 830417871 +frame_id: "camera_color_optical_frame" +[INFO] [1735697145.243676]: received mask_obs with header seq: 909 +stamp: + secs: 1735697145 + nsecs: 246792554 +frame_id: '' and grasped_object_mask header seq: 133468 +stamp: + secs: 1735697145 + nsecs: 30255318 +frame_id: "camera_color_optical_frame" +[INFO] [1735697145.443752]: received mask_obs with header seq: 910 +stamp: + secs: 1735697145 + nsecs: 446915864 +frame_id: '' and grasped_object_mask header seq: 133474 +stamp: + secs: 1735697145 + nsecs: 230092287 +frame_id: "camera_color_optical_frame" +[INFO] [1735697145.649808]: received mask_obs with header seq: 911 +stamp: + secs: 1735697145 + nsecs: 653177022 +frame_id: '' and grasped_object_mask header seq: 133480 +stamp: + secs: 1735697145 + nsecs: 429927349 +frame_id: "camera_color_optical_frame" +[INFO] [1735697145.848867]: received mask_obs with header seq: 912 +stamp: + secs: 1735697145 + nsecs: 852311372 +frame_id: '' and grasped_object_mask header seq: 133486 +stamp: + secs: 1735697145 + nsecs: 629761219 +frame_id: "camera_color_optical_frame" +[INFO] [1735697146.049898]: received mask_obs with header seq: 913 +stamp: + secs: 1735697146 + nsecs: 53177833 +frame_id: '' and grasped_object_mask header seq: 133492 +stamp: + secs: 1735697145 + nsecs: 829595089 +frame_id: "camera_color_optical_frame" +[INFO] [1735697146.249327]: received mask_obs with header seq: 914 +stamp: + secs: 1735697146 + nsecs: 252746582 +frame_id: '' and grasped_object_mask header seq: 133498 +stamp: + secs: 1735697146 + nsecs: 29428482 +frame_id: "camera_color_optical_frame" +[INFO] [1735697146.449411]: received mask_obs with header seq: 915 +stamp: + secs: 1735697146 + nsecs: 452652692 +frame_id: '' and grasped_object_mask header seq: 133504 +stamp: + secs: 1735697146 + nsecs: 229262829 +frame_id: "camera_color_optical_frame" +[INFO] [1735697146.648673]: received mask_obs with header seq: 916 +stamp: + secs: 1735697146 + nsecs: 651796579 +frame_id: '' and grasped_object_mask header seq: 133510 +stamp: + secs: 1735697146 + nsecs: 429101944 +frame_id: "camera_color_optical_frame" +[INFO] [1735697146.849419]: received mask_obs with header seq: 917 +stamp: + secs: 1735697146 + nsecs: 852834224 +frame_id: '' and grasped_object_mask header seq: 133515 +stamp: + secs: 1735697146 + nsecs: 595635414 +frame_id: "camera_color_optical_frame" +[INFO] [1735697147.049031]: received mask_obs with header seq: 918 +stamp: + secs: 1735697147 + nsecs: 52455186 +frame_id: '' and grasped_object_mask header seq: 133522 +stamp: + secs: 1735697146 + nsecs: 828782082 +frame_id: "camera_color_optical_frame" +[INFO] [1735697147.247771]: received mask_obs with header seq: 919 +stamp: + secs: 1735697147 + nsecs: 251193284 +frame_id: '' and grasped_object_mask header seq: 133528 +stamp: + secs: 1735697147 + nsecs: 28621435 +frame_id: "camera_color_optical_frame" +[INFO] [1735697147.449640]: received mask_obs with header seq: 920 +stamp: + secs: 1735697147 + nsecs: 453022956 +frame_id: '' and grasped_object_mask header seq: 133534 +stamp: + secs: 1735697147 + nsecs: 228460073 +frame_id: "camera_color_optical_frame" +[INFO] [1735697147.649927]: received mask_obs with header seq: 921 +stamp: + secs: 1735697147 + nsecs: 653132915 +frame_id: '' and grasped_object_mask header seq: 133540 +stamp: + secs: 1735697147 + nsecs: 428293228 +frame_id: "camera_color_optical_frame" +[INFO] [1735697147.848045]: received mask_obs with header seq: 922 +stamp: + secs: 1735697147 + nsecs: 851426601 +frame_id: '' and grasped_object_mask header seq: 133546 +stamp: + secs: 1735697147 + nsecs: 628126144 +frame_id: "camera_color_optical_frame" +[INFO] [1735697148.048839]: received mask_obs with header seq: 923 +stamp: + secs: 1735697148 + nsecs: 52282571 +frame_id: '' and grasped_object_mask header seq: 133552 +stamp: + secs: 1735697147 + nsecs: 827960014 +frame_id: "camera_color_optical_frame" +[INFO] [1735697148.253891]: received mask_obs with header seq: 924 +stamp: + secs: 1735697148 + nsecs: 257185697 +frame_id: '' and grasped_object_mask header seq: 133558 +stamp: + secs: 1735697148 + nsecs: 27792215 +frame_id: "camera_color_optical_frame" +[INFO] [1735697148.448879]: received mask_obs with header seq: 925 +stamp: + secs: 1735697148 + nsecs: 452314853 +frame_id: '' and grasped_object_mask header seq: 133564 +stamp: + secs: 1735697148 + nsecs: 227625370 +frame_id: "camera_color_optical_frame" +[INFO] [1735697148.648541]: received mask_obs with header seq: 926 +stamp: + secs: 1735697148 + nsecs: 651968479 +frame_id: '' and grasped_object_mask header seq: 133570 +stamp: + secs: 1735697148 + nsecs: 427464962 +frame_id: "camera_color_optical_frame" +[INFO] [1735697148.848335]: received mask_obs with header seq: 927 +stamp: + secs: 1735697148 + nsecs: 851765394 +frame_id: '' and grasped_object_mask header seq: 133576 +stamp: + secs: 1735697148 + nsecs: 627304316 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.047767]: received mask_obs with header seq: 928 +stamp: + secs: 1735697149 + nsecs: 51222562 +frame_id: '' and grasped_object_mask header seq: 133582 +stamp: + secs: 1735697148 + nsecs: 827143669 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.247510]: received mask_obs with header seq: 929 +stamp: + secs: 1735697149 + nsecs: 250481128 +frame_id: '' and grasped_object_mask header seq: 133588 +stamp: + secs: 1735697149 + nsecs: 26983976 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.446760]: received mask_obs with header seq: 930 +stamp: + secs: 1735697149 + nsecs: 449872016 +frame_id: '' and grasped_object_mask header seq: 133594 +stamp: + secs: 1735697149 + nsecs: 226823092 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.647884]: received mask_obs with header seq: 931 +stamp: + secs: 1735697149 + nsecs: 651227474 +frame_id: '' and grasped_object_mask header seq: 133600 +stamp: + secs: 1735697149 + nsecs: 426661968 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.847656]: received mask_obs with header seq: 932 +stamp: + secs: 1735697149 + nsecs: 851090431 +frame_id: '' and grasped_object_mask header seq: 133606 +stamp: + secs: 1735697149 + nsecs: 626501083 +frame_id: "camera_color_optical_frame" +[INFO] [1735697149.875248]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735697150.675430]: Stored episode 3. +[INFO] [1735697150.676831]: Saving video... +[INFO] [1735697150.885067]: Video saved! +Starting episode 3 +[INFO] [1735697150.931603]: resetting environment +[INFO] [1735697150.950256]: cleared current plan +[INFO] [1735697150.958554]: moving to home +[INFO] [1735697154.564262]: reached home +[INFO] [1735697154.568453]: reset action history +[INFO] [1735697157.347214]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735697158.858510]: resetting environment +[INFO] [1735697158.870186]: cleared current plan +[INFO] [1735697158.872509]: moving to home +[INFO] [1735697159.876455]: reached home +[INFO] [1735697159.878748]: reset action history +[INFO] [1735697162.789644]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/episode_rosbags/episode_3_2024-12-31-21-06-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1740046.log +started roslaunch server http://158.130.50.37:36995/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1740772] +process[print_text-11]: started with pid [1740773] +process[rosbag_record-12]: started with pid [1740797] +started bagging! +[INFO] [1735697163.470505]: received mask_obs with header seq: 933 +stamp: + secs: 1735697163 + nsecs: 473626136 +frame_id: '' and grasped_object_mask header seq: 134016 +stamp: + secs: 1735697163 + nsecs: 282098532 +frame_id: "camera_color_optical_frame" +[INFO] [1735697163.679870]: received mask_obs with header seq: 934 +stamp: + secs: 1735697163 + nsecs: 682940959 +frame_id: '' and grasped_object_mask header seq: 134022 +stamp: + secs: 1735697163 + nsecs: 481943607 +frame_id: "camera_color_optical_frame" +[INFO] [1735697163.879616]: received mask_obs with header seq: 935 +stamp: + secs: 1735697163 + nsecs: 882958412 +frame_id: '' and grasped_object_mask header seq: 134028 +stamp: + secs: 1735697163 + nsecs: 681788683 +frame_id: "camera_color_optical_frame" +[INFO] [1735697164.084976]: received mask_obs with header seq: 936 +stamp: + secs: 1735697164 + nsecs: 88434696 +frame_id: '' and grasped_object_mask header seq: 134034 +stamp: + secs: 1735697163 + nsecs: 881634712 +frame_id: "camera_color_optical_frame" +[INFO] [1735697164.293545]: received mask_obs with header seq: 937 +stamp: + secs: 1735697164 + nsecs: 296964406 +frame_id: '' and grasped_object_mask header seq: 134040 +stamp: + secs: 1735697164 + nsecs: 81480265 +frame_id: "camera_color_optical_frame" +[INFO] [1735697164.492263]: received mask_obs with header seq: 938 +stamp: + secs: 1735697164 + nsecs: 495640754 +frame_id: '' and grasped_object_mask header seq: 134046 +stamp: + secs: 1735697164 + nsecs: 281325340 +frame_id: "camera_color_optical_frame" +[INFO] [1735697164.697321]: received mask_obs with header seq: 939 +stamp: + secs: 1735697164 + nsecs: 700684309 +frame_id: '' and grasped_object_mask header seq: 134052 +stamp: + secs: 1735697164 + nsecs: 481169462 +frame_id: "camera_color_optical_frame" +[INFO] [1735697164.901673]: received mask_obs with header seq: 940 +stamp: + secs: 1735697164 + nsecs: 905060529 +frame_id: '' and grasped_object_mask header seq: 134058 +stamp: + secs: 1735697164 + nsecs: 681013823 +frame_id: "camera_color_optical_frame" +[INFO] [1735697165.102664]: received mask_obs with header seq: 941 +stamp: + secs: 1735697165 + nsecs: 105703830 +frame_id: '' and grasped_object_mask header seq: 134064 +stamp: + secs: 1735697164 + nsecs: 880858898 +frame_id: "camera_color_optical_frame" +[INFO] [1735697165.300978]: received mask_obs with header seq: 942 +stamp: + secs: 1735697165 + nsecs: 304386377 +frame_id: '' and grasped_object_mask header seq: 134070 +stamp: + secs: 1735697165 + nsecs: 80704927 +frame_id: "camera_color_optical_frame" +[INFO] [1735697165.501149]: received mask_obs with header seq: 943 +stamp: + secs: 1735697165 + nsecs: 504530191 +frame_id: '' and grasped_object_mask header seq: 134076 +stamp: + secs: 1735697165 + nsecs: 280547619 +frame_id: "camera_color_optical_frame" +[INFO] [1735697165.701419]: received mask_obs with header seq: 944 +stamp: + secs: 1735697165 + nsecs: 704658031 +frame_id: '' and grasped_object_mask header seq: 134082 +stamp: + secs: 1735697165 + nsecs: 480384827 +frame_id: "camera_color_optical_frame" +[INFO] [1735697165.903352]: received mask_obs with header seq: 945 +stamp: + secs: 1735697165 + nsecs: 906787157 +frame_id: '' and grasped_object_mask header seq: 134088 +stamp: + secs: 1735697165 + nsecs: 680222034 +frame_id: "camera_color_optical_frame" +[INFO] [1735697166.103552]: received mask_obs with header seq: 946 +stamp: + secs: 1735697166 + nsecs: 106970310 +frame_id: '' and grasped_object_mask header seq: 134094 +stamp: + secs: 1735697165 + nsecs: 880059242 +frame_id: "camera_color_optical_frame" +[INFO] [1735697166.304247]: received mask_obs with header seq: 947 +stamp: + secs: 1735697166 + nsecs: 307589769 +frame_id: '' and grasped_object_mask header seq: 134100 +stamp: + secs: 1735697166 + nsecs: 79896450 +frame_id: "camera_color_optical_frame" +[INFO] [1735697166.504136]: received mask_obs with header seq: 948 +stamp: + secs: 1735697166 + nsecs: 507188320 +frame_id: '' and grasped_object_mask header seq: 134106 +stamp: + secs: 1735697166 + nsecs: 279736042 +frame_id: "camera_color_optical_frame" +[INFO] [1735697166.702041]: received mask_obs with header seq: 949 +stamp: + secs: 1735697166 + nsecs: 705455780 +frame_id: '' and grasped_object_mask header seq: 134112 +stamp: + secs: 1735697166 + nsecs: 479577065 +frame_id: "camera_color_optical_frame" +[INFO] [1735697166.901850]: received mask_obs with header seq: 950 +stamp: + secs: 1735697166 + nsecs: 905103445 +frame_id: '' and grasped_object_mask header seq: 134118 +stamp: + secs: 1735697166 + nsecs: 679418325 +frame_id: "camera_color_optical_frame" +[INFO] [1735697167.104942]: received mask_obs with header seq: 951 +stamp: + secs: 1735697167 + nsecs: 108401060 +frame_id: '' and grasped_object_mask header seq: 134124 +stamp: + secs: 1735697166 + nsecs: 879259348 +frame_id: "camera_color_optical_frame" +[INFO] [1735697167.302711]: received mask_obs with header seq: 952 +stamp: + secs: 1735697167 + nsecs: 305675029 +frame_id: '' and grasped_object_mask header seq: 134130 +stamp: + secs: 1735697167 + nsecs: 79100132 +frame_id: "camera_color_optical_frame" +[INFO] [1735697167.502471]: received mask_obs with header seq: 953 +stamp: + secs: 1735697167 + nsecs: 505872726 +frame_id: '' and grasped_object_mask header seq: 134136 +stamp: + secs: 1735697167 + nsecs: 278942108 +frame_id: "camera_color_optical_frame" +[INFO] [1735697167.700605]: received mask_obs with header seq: 954 +stamp: + secs: 1735697167 + nsecs: 704034566 +frame_id: '' and grasped_object_mask header seq: 134142 +stamp: + secs: 1735697167 + nsecs: 478786707 +frame_id: "camera_color_optical_frame" +[INFO] [1735697167.902246]: received mask_obs with header seq: 955 +stamp: + secs: 1735697167 + nsecs: 905669689 +frame_id: '' and grasped_object_mask header seq: 134148 +stamp: + secs: 1735697167 + nsecs: 678632259 +frame_id: "camera_color_optical_frame" +[INFO] [1735697168.103901]: received mask_obs with header seq: 956 +stamp: + secs: 1735697168 + nsecs: 107288837 +frame_id: '' and grasped_object_mask header seq: 134154 +stamp: + secs: 1735697167 + nsecs: 878477097 +frame_id: "camera_color_optical_frame" +[INFO] [1735697168.301927]: received mask_obs with header seq: 957 +stamp: + secs: 1735697168 + nsecs: 305185317 +frame_id: '' and grasped_object_mask header seq: 134160 +stamp: + secs: 1735697168 + nsecs: 78322172 +frame_id: "camera_color_optical_frame" +[INFO] [1735697168.502111]: received mask_obs with header seq: 958 +stamp: + secs: 1735697168 + nsecs: 505154848 +frame_id: '' and grasped_object_mask header seq: 134166 +stamp: + secs: 1735697168 + nsecs: 278166533 +frame_id: "camera_color_optical_frame" +[INFO] [1735697168.700251]: received mask_obs with header seq: 959 +stamp: + secs: 1735697168 + nsecs: 703191757 +frame_id: '' and grasped_object_mask header seq: 134172 +stamp: + secs: 1735697168 + nsecs: 478008270 +frame_id: "camera_color_optical_frame" +[INFO] [1735697168.900944]: received mask_obs with header seq: 960 +stamp: + secs: 1735697168 + nsecs: 904307365 +frame_id: '' and grasped_object_mask header seq: 134178 +stamp: + secs: 1735697168 + nsecs: 677850246 +frame_id: "camera_color_optical_frame" +[INFO] [1735697169.104158]: received mask_obs with header seq: 961 +stamp: + secs: 1735697169 + nsecs: 107184410 +frame_id: '' and grasped_object_mask header seq: 134184 +stamp: + secs: 1735697168 + nsecs: 877693415 +frame_id: "camera_color_optical_frame" +[INFO] [1735697169.300615]: received mask_obs with header seq: 962 +stamp: + secs: 1735697169 + nsecs: 304043292 +frame_id: '' and grasped_object_mask header seq: 134190 +stamp: + secs: 1735697169 + nsecs: 77535391 +frame_id: "camera_color_optical_frame" +[INFO] [1735697169.499438]: received mask_obs with header seq: 963 +stamp: + secs: 1735697169 + nsecs: 502782106 +frame_id: '' and grasped_object_mask header seq: 134196 +stamp: + secs: 1735697169 + nsecs: 277375698 +frame_id: "camera_color_optical_frame" +[INFO] [1735697169.701111]: received mask_obs with header seq: 964 +stamp: + secs: 1735697169 + nsecs: 704509019 +frame_id: '' and grasped_object_mask header seq: 134202 +stamp: + secs: 1735697169 + nsecs: 477212667 +frame_id: "camera_color_optical_frame" +[INFO] [1735697169.900119]: received mask_obs with header seq: 965 +stamp: + secs: 1735697169 + nsecs: 903556585 +frame_id: '' and grasped_object_mask header seq: 134208 +stamp: + secs: 1735697169 + nsecs: 677049398 +frame_id: "camera_color_optical_frame" +[INFO] [1735697170.101558]: received mask_obs with header seq: 966 +stamp: + secs: 1735697170 + nsecs: 104918003 +frame_id: '' and grasped_object_mask header seq: 134214 +stamp: + secs: 1735697169 + nsecs: 876885653 +frame_id: "camera_color_optical_frame" +[INFO] [1735697170.298780]: received mask_obs with header seq: 967 +stamp: + secs: 1735697170 + nsecs: 302204608 +frame_id: '' and grasped_object_mask header seq: 134220 +stamp: + secs: 1735697170 + nsecs: 76723099 +frame_id: "camera_color_optical_frame" +[INFO] [1735697170.498636]: received mask_obs with header seq: 968 +stamp: + secs: 1735697170 + nsecs: 501919031 +frame_id: '' and grasped_object_mask header seq: 134226 +stamp: + secs: 1735697170 + nsecs: 276560545 +frame_id: "camera_color_optical_frame" +[INFO] [1735697170.697956]: received mask_obs with header seq: 969 +stamp: + secs: 1735697170 + nsecs: 701403379 +frame_id: '' and grasped_object_mask header seq: 134232 +stamp: + secs: 1735697170 + nsecs: 476400137 +frame_id: "camera_color_optical_frame" +[INFO] [1735697170.900364]: received mask_obs with header seq: 970 +stamp: + secs: 1735697170 + nsecs: 903397560 +frame_id: '' and grasped_object_mask header seq: 134238 +stamp: + secs: 1735697170 + nsecs: 676239729 +frame_id: "camera_color_optical_frame" +[INFO] [1735697171.099788]: received mask_obs with header seq: 971 +stamp: + secs: 1735697171 + nsecs: 103012323 +frame_id: '' and grasped_object_mask header seq: 134244 +stamp: + secs: 1735697170 + nsecs: 876079559 +frame_id: "camera_color_optical_frame" +[INFO] [1735697171.298464]: received mask_obs with header seq: 972 +stamp: + secs: 1735697171 + nsecs: 301696300 +frame_id: '' and grasped_object_mask header seq: 134250 +stamp: + secs: 1735697171 + nsecs: 75919390 +frame_id: "camera_color_optical_frame" +[INFO] [1735697171.498739]: received mask_obs with header seq: 973 +stamp: + secs: 1735697171 + nsecs: 502088308 +frame_id: '' and grasped_object_mask header seq: 134256 +stamp: + secs: 1735697171 + nsecs: 275757551 +frame_id: "camera_color_optical_frame" +[INFO] [1735697171.697759]: received mask_obs with header seq: 974 +stamp: + secs: 1735697171 + nsecs: 701008319 +frame_id: '' and grasped_object_mask header seq: 134262 +stamp: + secs: 1735697171 + nsecs: 475589275 +frame_id: "camera_color_optical_frame" +[INFO] [1735697171.899232]: received mask_obs with header seq: 975 +stamp: + secs: 1735697171 + nsecs: 902490615 +frame_id: '' and grasped_object_mask header seq: 134268 +stamp: + secs: 1735697171 + nsecs: 675420761 +frame_id: "camera_color_optical_frame" +[INFO] [1735697172.101572]: received mask_obs with header seq: 976 +stamp: + secs: 1735697172 + nsecs: 103792667 +frame_id: '' and grasped_object_mask header seq: 134274 +stamp: + secs: 1735697171 + nsecs: 875251532 +frame_id: "camera_color_optical_frame" +[INFO] [1735697172.303338]: received mask_obs with header seq: 977 +stamp: + secs: 1735697172 + nsecs: 306586265 +frame_id: '' and grasped_object_mask header seq: 134280 +stamp: + secs: 1735697172 + nsecs: 75082064 +frame_id: "camera_color_optical_frame" +[INFO] [1735697172.501440]: received mask_obs with header seq: 978 +stamp: + secs: 1735697172 + nsecs: 504763126 +frame_id: '' and grasped_object_mask header seq: 134286 +stamp: + secs: 1735697172 + nsecs: 274913788 +frame_id: "camera_color_optical_frame" +[INFO] [1735697172.698325]: received mask_obs with header seq: 979 +stamp: + secs: 1735697172 + nsecs: 701561689 +frame_id: '' and grasped_object_mask header seq: 134292 +stamp: + secs: 1735697172 + nsecs: 474749565 +frame_id: "camera_color_optical_frame" +[INFO] [1735697172.898258]: received mask_obs with header seq: 980 +stamp: + secs: 1735697172 + nsecs: 901404857 +frame_id: '' and grasped_object_mask header seq: 134298 +stamp: + secs: 1735697172 + nsecs: 674583673 +frame_id: "camera_color_optical_frame" +[INFO] [1735697173.106220]: received mask_obs with header seq: 981 +stamp: + secs: 1735697173 + nsecs: 109596729 +frame_id: '' and grasped_object_mask header seq: 134304 +stamp: + secs: 1735697172 + nsecs: 874418020 +frame_id: "camera_color_optical_frame" +[INFO] [1735697173.303915]: received mask_obs with header seq: 982 +stamp: + secs: 1735697173 + nsecs: 306970834 +frame_id: '' and grasped_object_mask header seq: 134310 +stamp: + secs: 1735697173 + nsecs: 74251652 +frame_id: "camera_color_optical_frame" +[INFO] [1735697173.498303]: received mask_obs with header seq: 983 +stamp: + secs: 1735697173 + nsecs: 500722885 +frame_id: '' and grasped_object_mask header seq: 134316 +stamp: + secs: 1735697173 + nsecs: 274087429 +frame_id: "camera_color_optical_frame" +[INFO] [1735697173.697419]: received mask_obs with header seq: 984 +stamp: + secs: 1735697173 + nsecs: 700652360 +frame_id: '' and grasped_object_mask header seq: 134322 +stamp: + secs: 1735697173 + nsecs: 473927021 +frame_id: "camera_color_optical_frame" +[INFO] [1735697173.897753]: received mask_obs with header seq: 985 +stamp: + secs: 1735697173 + nsecs: 900903940 +frame_id: '' and grasped_object_mask header seq: 134328 +stamp: + secs: 1735697173 + nsecs: 673767567 +frame_id: "camera_color_optical_frame" +[INFO] [1735697174.098969]: received mask_obs with header seq: 986 +stamp: + secs: 1735697174 + nsecs: 102452754 +frame_id: '' and grasped_object_mask header seq: 134334 +stamp: + secs: 1735697173 + nsecs: 873607397 +frame_id: "camera_color_optical_frame" +[INFO] [1735697174.295621]: received mask_obs with header seq: 987 +stamp: + secs: 1735697174 + nsecs: 299010276 +frame_id: '' and grasped_object_mask header seq: 134340 +stamp: + secs: 1735697174 + nsecs: 73447227 +frame_id: "camera_color_optical_frame" +[INFO] [1735697174.499447]: received mask_obs with header seq: 988 +stamp: + secs: 1735697174 + nsecs: 502874374 +frame_id: '' and grasped_object_mask header seq: 134346 +stamp: + secs: 1735697174 + nsecs: 273284912 +frame_id: "camera_color_optical_frame" +[INFO] [1735697174.698980]: received mask_obs with header seq: 989 +stamp: + secs: 1735697174 + nsecs: 702198982 +frame_id: '' and grasped_object_mask header seq: 134352 +stamp: + secs: 1735697174 + nsecs: 473118305 +frame_id: "camera_color_optical_frame" +[INFO] [1735697174.896841]: received mask_obs with header seq: 990 +stamp: + secs: 1735697174 + nsecs: 899733304 +frame_id: '' and grasped_object_mask header seq: 134358 +stamp: + secs: 1735697174 + nsecs: 672951460 +frame_id: "camera_color_optical_frame" +[INFO] [1735697175.096984]: received mask_obs with header seq: 991 +stamp: + secs: 1735697175 + nsecs: 100315809 +frame_id: '' and grasped_object_mask header seq: 134364 +stamp: + secs: 1735697174 + nsecs: 872784376 +frame_id: "camera_color_optical_frame" +[INFO] [1735697175.295527]: received mask_obs with header seq: 992 +stamp: + secs: 1735697175 + nsecs: 298599720 +frame_id: '' and grasped_object_mask header seq: 134370 +stamp: + secs: 1735697175 + nsecs: 72618246 +frame_id: "camera_color_optical_frame" +[INFO] [1735697175.494396]: received mask_obs with header seq: 993 +stamp: + secs: 1735697175 + nsecs: 497871637 +frame_id: '' and grasped_object_mask header seq: 134376 +stamp: + secs: 1735697175 + nsecs: 272454023 +frame_id: "camera_color_optical_frame" +[INFO] [1735697175.705350]: received mask_obs with header seq: 994 +stamp: + secs: 1735697175 + nsecs: 708726644 +frame_id: '' and grasped_object_mask header seq: 134382 +stamp: + secs: 1735697175 + nsecs: 472297430 +frame_id: "camera_color_optical_frame" +[INFO] [1735697175.910580]: received mask_obs with header seq: 995 +stamp: + secs: 1735697175 + nsecs: 913998603 +frame_id: '' and grasped_object_mask header seq: 134388 +stamp: + secs: 1735697175 + nsecs: 672141075 +frame_id: "camera_color_optical_frame" +[INFO] [1735697176.108607]: received mask_obs with header seq: 996 +stamp: + secs: 1735697176 + nsecs: 112035036 +frame_id: '' and grasped_object_mask header seq: 134394 +stamp: + secs: 1735697175 + nsecs: 871984959 +frame_id: "camera_color_optical_frame" +[INFO] [1735697176.311430]: received mask_obs with header seq: 997 +stamp: + secs: 1735697176 + nsecs: 314884662 +frame_id: '' and grasped_object_mask header seq: 134400 +stamp: + secs: 1735697176 + nsecs: 71828604 +frame_id: "camera_color_optical_frame" +[INFO] [1735697176.509221]: received mask_obs with header seq: 998 +stamp: + secs: 1735697176 + nsecs: 512585639 +frame_id: '' and grasped_object_mask header seq: 134406 +stamp: + secs: 1735697176 + nsecs: 271672964 +frame_id: "camera_color_optical_frame" +[INFO] [1735697176.704517]: received mask_obs with header seq: 999 +stamp: + secs: 1735697176 + nsecs: 707173109 +frame_id: '' and grasped_object_mask header seq: 134412 +stamp: + secs: 1735697176 + nsecs: 471513748 +frame_id: "camera_color_optical_frame" +[INFO] [1735697176.899469]: received mask_obs with header seq: 1000 +stamp: + secs: 1735697176 + nsecs: 902950048 +frame_id: '' and grasped_object_mask header seq: 134418 +stamp: + secs: 1735697176 + nsecs: 671354771 +frame_id: "camera_color_optical_frame" +[INFO] [1735697177.097877]: received mask_obs with header seq: 1001 +stamp: + secs: 1735697177 + nsecs: 101318120 +frame_id: '' and grasped_object_mask header seq: 134424 +stamp: + secs: 1735697176 + nsecs: 871195793 +frame_id: "camera_color_optical_frame" +[INFO] [1735697177.296750]: received mask_obs with header seq: 1002 +stamp: + secs: 1735697177 + nsecs: 299467563 +frame_id: '' and grasped_object_mask header seq: 134430 +stamp: + secs: 1735697177 + nsecs: 71037054 +frame_id: "camera_color_optical_frame" +[INFO] [1735697177.497800]: received mask_obs with header seq: 1003 +stamp: + secs: 1735697177 + nsecs: 500859737 +frame_id: '' and grasped_object_mask header seq: 134436 +stamp: + secs: 1735697177 + nsecs: 270876646 +frame_id: "camera_color_optical_frame" +[INFO] [1735697177.699458]: received mask_obs with header seq: 1004 +stamp: + secs: 1735697177 + nsecs: 702930688 +frame_id: '' and grasped_object_mask header seq: 134442 +stamp: + secs: 1735697177 + nsecs: 470714569 +frame_id: "camera_color_optical_frame" +[INFO] [1735697177.897601]: received mask_obs with header seq: 1005 +stamp: + secs: 1735697177 + nsecs: 900730371 +frame_id: '' and grasped_object_mask header seq: 134448 +stamp: + secs: 1735697177 + nsecs: 670551538 +frame_id: "camera_color_optical_frame" +[INFO] [1735697178.098858]: received mask_obs with header seq: 1006 +stamp: + secs: 1735697178 + nsecs: 102141380 +frame_id: '' and grasped_object_mask header seq: 134454 +stamp: + secs: 1735697177 + nsecs: 870388031 +frame_id: "camera_color_optical_frame" +[INFO] [1735697178.298109]: received mask_obs with header seq: 1007 +stamp: + secs: 1735697178 + nsecs: 301451206 +frame_id: '' and grasped_object_mask header seq: 134460 +stamp: + secs: 1735697178 + nsecs: 70224285 +frame_id: "camera_color_optical_frame" +[INFO] [1735697178.496997]: received mask_obs with header seq: 1008 +stamp: + secs: 1735697178 + nsecs: 500043869 +frame_id: '' and grasped_object_mask header seq: 134466 +stamp: + secs: 1735697178 + nsecs: 270062923 +frame_id: "camera_color_optical_frame" +[INFO] [1735697178.697796]: received mask_obs with header seq: 1009 +stamp: + secs: 1735697178 + nsecs: 701217651 +frame_id: '' and grasped_object_mask header seq: 134472 +stamp: + secs: 1735697178 + nsecs: 469906807 +frame_id: "camera_color_optical_frame" +[INFO] [1735697178.896604]: received mask_obs with header seq: 1010 +stamp: + secs: 1735697178 + nsecs: 899342298 +frame_id: '' and grasped_object_mask header seq: 134478 +stamp: + secs: 1735697178 + nsecs: 669751406 +frame_id: "camera_color_optical_frame" +[INFO] [1735697179.098233]: received mask_obs with header seq: 1011 +stamp: + secs: 1735697179 + nsecs: 101720809 +frame_id: '' and grasped_object_mask header seq: 134484 +stamp: + secs: 1735697178 + nsecs: 869595528 +frame_id: "camera_color_optical_frame" +[INFO] [1735697179.296470]: received mask_obs with header seq: 1012 +stamp: + secs: 1735697179 + nsecs: 299935579 +frame_id: '' and grasped_object_mask header seq: 134490 +stamp: + secs: 1735697179 + nsecs: 69439411 +frame_id: "camera_color_optical_frame" +[INFO] [1735697179.496649]: received mask_obs with header seq: 1013 +stamp: + secs: 1735697179 + nsecs: 500124931 +frame_id: '' and grasped_object_mask header seq: 134496 +stamp: + secs: 1735697179 + nsecs: 269280910 +frame_id: "camera_color_optical_frame" +[INFO] [1735697179.697029]: received mask_obs with header seq: 1014 +stamp: + secs: 1735697179 + nsecs: 700263500 +frame_id: '' and grasped_object_mask header seq: 134502 +stamp: + secs: 1735697179 + nsecs: 469116926 +frame_id: "camera_color_optical_frame" +[INFO] [1735697179.895649]: received mask_obs with header seq: 1015 +stamp: + secs: 1735697179 + nsecs: 898591279 +frame_id: '' and grasped_object_mask header seq: 134508 +stamp: + secs: 1735697179 + nsecs: 668952942 +frame_id: "camera_color_optical_frame" +[INFO] [1735697180.098495]: received mask_obs with header seq: 1016 +stamp: + secs: 1735697180 + nsecs: 101470947 +frame_id: '' and grasped_object_mask header seq: 134514 +stamp: + secs: 1735697179 + nsecs: 868789911 +frame_id: "camera_color_optical_frame" +[INFO] [1735697180.295663]: received mask_obs with header seq: 1017 +stamp: + secs: 1735697180 + nsecs: 299109220 +frame_id: '' and grasped_object_mask header seq: 134520 +stamp: + secs: 1735697180 + nsecs: 68625689 +frame_id: "camera_color_optical_frame" +[INFO] [1735697180.496856]: received mask_obs with header seq: 1018 +stamp: + secs: 1735697180 + nsecs: 500225305 +frame_id: '' and grasped_object_mask header seq: 134526 +stamp: + secs: 1735697180 + nsecs: 268463612 +frame_id: "camera_color_optical_frame" +[INFO] [1735697180.698888]: received mask_obs with header seq: 1019 +stamp: + secs: 1735697180 + nsecs: 702298641 +frame_id: '' and grasped_object_mask header seq: 134532 +stamp: + secs: 1735697180 + nsecs: 468305111 +frame_id: "camera_color_optical_frame" +[INFO] [1735697180.898096]: received mask_obs with header seq: 1020 +stamp: + secs: 1735697180 + nsecs: 901552438 +frame_id: '' and grasped_object_mask header seq: 134538 +stamp: + secs: 1735697180 + nsecs: 668147087 +frame_id: "camera_color_optical_frame" +[INFO] [1735697181.097706]: received mask_obs with header seq: 1021 +stamp: + secs: 1735697181 + nsecs: 101179361 +frame_id: '' and grasped_object_mask header seq: 134544 +stamp: + secs: 1735697180 + nsecs: 867988825 +frame_id: "camera_color_optical_frame" +[INFO] [1735697181.297261]: received mask_obs with header seq: 1022 +stamp: + secs: 1735697181 + nsecs: 300649404 +frame_id: '' and grasped_object_mask header seq: 134550 +stamp: + secs: 1735697181 + nsecs: 67831755 +frame_id: "camera_color_optical_frame" +[INFO] [1735697181.497077]: received mask_obs with header seq: 1023 +stamp: + secs: 1735697181 + nsecs: 500174283 +frame_id: '' and grasped_object_mask header seq: 134556 +stamp: + secs: 1735697181 + nsecs: 267672539 +frame_id: "camera_color_optical_frame" +[INFO] [1735697181.695783]: received mask_obs with header seq: 1024 +stamp: + secs: 1735697181 + nsecs: 699195623 +frame_id: '' and grasped_object_mask header seq: 134562 +stamp: + secs: 1735697181 + nsecs: 467509985 +frame_id: "camera_color_optical_frame" +[INFO] [1735697181.896511]: received mask_obs with header seq: 1025 +stamp: + secs: 1735697181 + nsecs: 899987936 +frame_id: '' and grasped_object_mask header seq: 134568 +stamp: + secs: 1735697181 + nsecs: 667347193 +frame_id: "camera_color_optical_frame" +[INFO] [1735697182.097577]: received mask_obs with header seq: 1026 +stamp: + secs: 1735697182 + nsecs: 101047277 +frame_id: '' and grasped_object_mask header seq: 134574 +stamp: + secs: 1735697181 + nsecs: 867184401 +frame_id: "camera_color_optical_frame" +[INFO] [1735697182.295464]: received mask_obs with header seq: 1027 +stamp: + secs: 1735697182 + nsecs: 298803091 +frame_id: '' and grasped_object_mask header seq: 134580 +stamp: + secs: 1735697182 + nsecs: 67021132 +frame_id: "camera_color_optical_frame" +[INFO] [1735697182.505497]: received mask_obs with header seq: 1028 +stamp: + secs: 1735697182 + nsecs: 508923530 +frame_id: '' and grasped_object_mask header seq: 134586 +stamp: + secs: 1735697182 + nsecs: 266860247 +frame_id: "camera_color_optical_frame" +[INFO] [1735697182.713780]: received mask_obs with header seq: 1029 +stamp: + secs: 1735697182 + nsecs: 716955661 +frame_id: '' and grasped_object_mask header seq: 134592 +stamp: + secs: 1735697182 + nsecs: 466704845 +frame_id: "camera_color_optical_frame" +[INFO] [1735697182.908313]: received mask_obs with header seq: 1030 +stamp: + secs: 1735697182 + nsecs: 911750555 +frame_id: '' and grasped_object_mask header seq: 134598 +stamp: + secs: 1735697182 + nsecs: 666548491 +frame_id: "camera_color_optical_frame" +[INFO] [1735697183.103944]: received mask_obs with header seq: 1031 +stamp: + secs: 1735697183 + nsecs: 107357740 +frame_id: '' and grasped_object_mask header seq: 134604 +stamp: + secs: 1735697182 + nsecs: 866392612 +frame_id: "camera_color_optical_frame" +[INFO] [1735697183.297245]: received mask_obs with header seq: 1032 +stamp: + secs: 1735697183 + nsecs: 300452470 +frame_id: '' and grasped_object_mask header seq: 134610 +stamp: + secs: 1735697183 + nsecs: 66236973 +frame_id: "camera_color_optical_frame" +[INFO] [1735697183.497229]: received mask_obs with header seq: 1033 +stamp: + secs: 1735697183 + nsecs: 500701427 +frame_id: '' and grasped_object_mask header seq: 134616 +stamp: + secs: 1735697183 + nsecs: 266079903 +frame_id: "camera_color_optical_frame" +[INFO] [1735697183.701535]: received mask_obs with header seq: 1034 +stamp: + secs: 1735697183 + nsecs: 704884767 +frame_id: '' and grasped_object_mask header seq: 134622 +stamp: + secs: 1735697183 + nsecs: 465919018 +frame_id: "camera_color_optical_frame" +[INFO] [1735697183.896101]: received mask_obs with header seq: 1035 +stamp: + secs: 1735697183 + nsecs: 899563312 +frame_id: '' and grasped_object_mask header seq: 134628 +stamp: + secs: 1735697183 + nsecs: 665759325 +frame_id: "camera_color_optical_frame" +[INFO] [1735697184.097975]: received mask_obs with header seq: 1036 +stamp: + secs: 1735697184 + nsecs: 101283311 +frame_id: '' and grasped_object_mask header seq: 134634 +stamp: + secs: 1735697183 + nsecs: 865598440 +frame_id: "camera_color_optical_frame" +[INFO] [1735697184.297273]: received mask_obs with header seq: 1037 +stamp: + secs: 1735697184 + nsecs: 300736188 +frame_id: '' and grasped_object_mask header seq: 134640 +stamp: + secs: 1735697184 + nsecs: 65437794 +frame_id: "camera_color_optical_frame" +[INFO] [1735697184.496213]: received mask_obs with header seq: 1038 +stamp: + secs: 1735697184 + nsecs: 499673843 +frame_id: '' and grasped_object_mask header seq: 134646 +stamp: + secs: 1735697184 + nsecs: 265277147 +frame_id: "camera_color_optical_frame" +[INFO] [1735697184.697068]: received mask_obs with header seq: 1039 +stamp: + secs: 1735697184 + nsecs: 700494527 +frame_id: '' and grasped_object_mask header seq: 134652 +stamp: + secs: 1735697184 + nsecs: 467333555 +frame_id: "camera_color_optical_frame" +[INFO] [1735697184.897208]: received mask_obs with header seq: 1040 +stamp: + secs: 1735697184 + nsecs: 900652170 +frame_id: '' and grasped_object_mask header seq: 134658 +stamp: + secs: 1735697184 + nsecs: 669547319 +frame_id: "camera_color_optical_frame" +[INFO] [1735697185.098868]: received mask_obs with header seq: 1041 +stamp: + secs: 1735697185 + nsecs: 102262020 +frame_id: '' and grasped_object_mask header seq: 134664 +stamp: + secs: 1735697184 + nsecs: 871849060 +frame_id: "camera_color_optical_frame" +[INFO] [1735697185.296521]: received mask_obs with header seq: 1042 +stamp: + secs: 1735697185 + nsecs: 299984931 +frame_id: '' and grasped_object_mask header seq: 134670 +stamp: + secs: 1735697185 + nsecs: 74236631 +frame_id: "camera_color_optical_frame" +[INFO] [1735697185.496908]: received mask_obs with header seq: 1043 +stamp: + secs: 1735697185 + nsecs: 500355958 +frame_id: '' and grasped_object_mask header seq: 134676 +stamp: + secs: 1735697185 + nsecs: 276710987 +frame_id: "camera_color_optical_frame" +[INFO] [1735697185.697371]: received mask_obs with header seq: 1044 +stamp: + secs: 1735697185 + nsecs: 700744867 +frame_id: '' and grasped_object_mask header seq: 134682 +stamp: + secs: 1735697185 + nsecs: 476431370 +frame_id: "camera_color_optical_frame" +[INFO] [1735697185.896924]: received mask_obs with header seq: 1045 +stamp: + secs: 1735697185 + nsecs: 900227785 +frame_id: '' and grasped_object_mask header seq: 134688 +stamp: + secs: 1735697185 + nsecs: 676042080 +frame_id: "camera_color_optical_frame" +[INFO] [1735697186.097013]: received mask_obs with header seq: 1046 +stamp: + secs: 1735697186 + nsecs: 100504159 +frame_id: '' and grasped_object_mask header seq: 134694 +stamp: + secs: 1735697185 + nsecs: 875641108 +frame_id: "camera_color_optical_frame" +[INFO] [1735697186.296734]: received mask_obs with header seq: 1047 +stamp: + secs: 1735697186 + nsecs: 300199747 +frame_id: '' and grasped_object_mask header seq: 134700 +stamp: + secs: 1735697186 + nsecs: 75229645 +frame_id: "camera_color_optical_frame" +[INFO] [1735697186.496209]: received mask_obs with header seq: 1048 +stamp: + secs: 1735697186 + nsecs: 499634981 +frame_id: '' and grasped_object_mask header seq: 134706 +stamp: + secs: 1735697186 + nsecs: 274806261 +frame_id: "camera_color_optical_frame" +[INFO] [1735697186.696728]: received mask_obs with header seq: 1049 +stamp: + secs: 1735697186 + nsecs: 700228691 +frame_id: '' and grasped_object_mask header seq: 134712 +stamp: + secs: 1735697186 + nsecs: 474417686 +frame_id: "camera_color_optical_frame" +[INFO] [1735697186.905423]: received mask_obs with header seq: 1050 +stamp: + secs: 1735697186 + nsecs: 908769607 +frame_id: '' and grasped_object_mask header seq: 134718 +stamp: + secs: 1735697186 + nsecs: 674019575 +frame_id: "camera_color_optical_frame" +[INFO] [1735697187.123811]: received mask_obs with header seq: 1051 +stamp: + secs: 1735697187 + nsecs: 126239538 +frame_id: '' and grasped_object_mask header seq: 134724 +stamp: + secs: 1735697186 + nsecs: 873609543 +frame_id: "camera_color_optical_frame" +[INFO] [1735697187.308994]: received mask_obs with header seq: 1052 +stamp: + secs: 1735697187 + nsecs: 312451362 +frame_id: '' and grasped_object_mask header seq: 134730 +stamp: + secs: 1735697187 + nsecs: 73188066 +frame_id: "camera_color_optical_frame" +[INFO] [1735697187.504712]: received mask_obs with header seq: 1053 +stamp: + secs: 1735697187 + nsecs: 508197784 +frame_id: '' and grasped_object_mask header seq: 134736 +stamp: + secs: 1735697187 + nsecs: 272755861 +frame_id: "camera_color_optical_frame" +[INFO] [1735697187.696570]: received mask_obs with header seq: 1054 +stamp: + secs: 1735697187 + nsecs: 699904918 +frame_id: '' and grasped_object_mask header seq: 134742 +stamp: + secs: 1735697187 + nsecs: 472367287 +frame_id: "camera_color_optical_frame" +[INFO] [1735697187.896147]: received mask_obs with header seq: 1055 +stamp: + secs: 1735697187 + nsecs: 899649381 +frame_id: '' and grasped_object_mask header seq: 134748 +stamp: + secs: 1735697187 + nsecs: 671969175 +frame_id: "camera_color_optical_frame" +[INFO] [1735697188.097770]: received mask_obs with header seq: 1056 +stamp: + secs: 1735697188 + nsecs: 100893497 +frame_id: '' and grasped_object_mask header seq: 134754 +stamp: + secs: 1735697187 + nsecs: 871559381 +frame_id: "camera_color_optical_frame" +[INFO] [1735697188.297145]: received mask_obs with header seq: 1057 +stamp: + secs: 1735697188 + nsecs: 300506353 +frame_id: '' and grasped_object_mask header seq: 134760 +stamp: + secs: 1735697188 + nsecs: 71137667 +frame_id: "camera_color_optical_frame" +[INFO] [1735697188.496573]: received mask_obs with header seq: 1058 +stamp: + secs: 1735697188 + nsecs: 499884128 +frame_id: '' and grasped_object_mask header seq: 134766 +stamp: + secs: 1735697188 + nsecs: 270704269 +frame_id: "camera_color_optical_frame" +[INFO] [1735697188.695755]: received mask_obs with header seq: 1059 +stamp: + secs: 1735697188 + nsecs: 699148893 +frame_id: '' and grasped_object_mask header seq: 134772 +stamp: + secs: 1735697188 + nsecs: 470322132 +frame_id: "camera_color_optical_frame" +[INFO] [1735697188.905978]: received mask_obs with header seq: 1060 +stamp: + secs: 1735697188 + nsecs: 909390926 +frame_id: '' and grasped_object_mask header seq: 134778 +stamp: + secs: 1735697188 + nsecs: 669929743 +frame_id: "camera_color_optical_frame" +[INFO] [1735697189.107724]: received mask_obs with header seq: 1061 +stamp: + secs: 1735697189 + nsecs: 111210584 +frame_id: '' and grasped_object_mask header seq: 134784 +stamp: + secs: 1735697188 + nsecs: 869526148 +frame_id: "camera_color_optical_frame" +[INFO] [1735697189.296469]: received mask_obs with header seq: 1062 +stamp: + secs: 1735697189 + nsecs: 299689769 +frame_id: '' and grasped_object_mask header seq: 134790 +stamp: + secs: 1735697189 + nsecs: 69110870 +frame_id: "camera_color_optical_frame" +[INFO] [1735697189.495882]: received mask_obs with header seq: 1063 +stamp: + secs: 1735697189 + nsecs: 499261856 +frame_id: '' and grasped_object_mask header seq: 134796 +stamp: + secs: 1735697189 + nsecs: 268684149 +frame_id: "camera_color_optical_frame" +[INFO] [1735697189.696012]: received mask_obs with header seq: 1064 +stamp: + secs: 1735697189 + nsecs: 699509859 +frame_id: '' and grasped_object_mask header seq: 134802 +stamp: + secs: 1735697189 + nsecs: 468302488 +frame_id: "camera_color_optical_frame" +[INFO] [1735697189.748935]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735697189.896530]: received mask_obs with header seq: 1065 +stamp: + secs: 1735697189 + nsecs: 899432897 +frame_id: '' and grasped_object_mask header seq: 134808 +stamp: + secs: 1735697189 + nsecs: 667912006 +frame_id: "camera_color_optical_frame" +[WARN] [1735697191.090596]: Plan exhausted +[WARN] [1735697191.131652]: Plan exhausted +[WARN] [1735697191.170684]: Plan exhausted +[INFO] [1735697190.528905]: Stored episode 4. +[INFO] [1735697190.533665]: Saving video... +[INFO] [1735697191.117724]: Video saved! +Starting episode 4 +[INFO] [1735697191.164316]: resetting environment +[INFO] [1735697191.186066]: cleared current plan +[INFO] [1735697191.188436]: moving to home +[INFO] [1735697194.798726]: reached home +[INFO] [1735697194.818842]: reset action history +[INFO] [1735697197.612770]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735697207.552463]: resetting environment +[INFO] [1735697207.565322]: cleared current plan +[INFO] [1735697207.565536]: moving to home +[INFO] [1735697208.568068]: reached home +[INFO] [1735697208.570566]: reset action history +[INFO] [1735697211.349774]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/episode_rosbags/episode_4_2024-12-31-21-06-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1740046.log +started roslaunch server http://158.130.50.37:36273/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1740939] +process[print_text-14]: started with pid [1740940] +process[rosbag_record-15]: started with pid [1740941] +started bagging! +[INFO] [1735697212.283548]: received mask_obs with header seq: 1066 +stamp: + secs: 1735697212 + nsecs: 286783218 +frame_id: '' and grasped_object_mask header seq: 135481 +stamp: + secs: 1735697212 + nsecs: 72682142 +frame_id: "camera_color_optical_frame" +[INFO] [1735697212.483800]: received mask_obs with header seq: 1067 +stamp: + secs: 1735697212 + nsecs: 487001419 +frame_id: '' and grasped_object_mask header seq: 135487 +stamp: + secs: 1735697212 + nsecs: 272258282 +frame_id: "camera_color_optical_frame" +[INFO] [1735697212.683363]: received mask_obs with header seq: 1068 +stamp: + secs: 1735697212 + nsecs: 686872005 +frame_id: '' and grasped_object_mask header seq: 135493 +stamp: + secs: 1735697212 + nsecs: 471853256 +frame_id: "camera_color_optical_frame" +[INFO] [1735697212.888715]: received mask_obs with header seq: 1069 +stamp: + secs: 1735697212 + nsecs: 891982316 +frame_id: '' and grasped_object_mask header seq: 135499 +stamp: + secs: 1735697212 + nsecs: 671467781 +frame_id: "camera_color_optical_frame" +[INFO] [1735697213.096845]: received mask_obs with header seq: 1070 +stamp: + secs: 1735697213 + nsecs: 100316047 +frame_id: '' and grasped_object_mask header seq: 135505 +stamp: + secs: 1735697212 + nsecs: 871069908 +frame_id: "camera_color_optical_frame" +[INFO] [1735697213.304149]: received mask_obs with header seq: 1071 +stamp: + secs: 1735697213 + nsecs: 307623624 +frame_id: '' and grasped_object_mask header seq: 135511 +stamp: + secs: 1735697213 + nsecs: 70660114 +frame_id: "camera_color_optical_frame" +[INFO] [1735697213.504689]: received mask_obs with header seq: 1072 +stamp: + secs: 1735697213 + nsecs: 508128404 +frame_id: '' and grasped_object_mask header seq: 135517 +stamp: + secs: 1735697213 + nsecs: 270239115 +frame_id: "camera_color_optical_frame" +[INFO] [1735697213.706909]: received mask_obs with header seq: 1073 +stamp: + secs: 1735697213 + nsecs: 710404634 +frame_id: '' and grasped_object_mask header seq: 135523 +stamp: + secs: 1735697213 + nsecs: 469836235 +frame_id: "camera_color_optical_frame" +[INFO] [1735697213.908094]: received mask_obs with header seq: 1074 +stamp: + secs: 1735697213 + nsecs: 911554098 +frame_id: '' and grasped_object_mask header seq: 135529 +stamp: + secs: 1735697213 + nsecs: 669451952 +frame_id: "camera_color_optical_frame" +[INFO] [1735697214.110160]: received mask_obs with header seq: 1075 +stamp: + secs: 1735697214 + nsecs: 113259077 +frame_id: '' and grasped_object_mask header seq: 135535 +stamp: + secs: 1735697213 + nsecs: 869057178 +frame_id: "camera_color_optical_frame" +[INFO] [1735697214.306981]: received mask_obs with header seq: 1076 +stamp: + secs: 1735697214 + nsecs: 310379505 +frame_id: '' and grasped_object_mask header seq: 135541 +stamp: + secs: 1735697214 + nsecs: 68649769 +frame_id: "camera_color_optical_frame" +[INFO] [1735697214.508185]: received mask_obs with header seq: 1077 +stamp: + secs: 1735697214 + nsecs: 511605739 +frame_id: '' and grasped_object_mask header seq: 135547 +stamp: + secs: 1735697214 + nsecs: 268230677 +frame_id: "camera_color_optical_frame" +[INFO] [1735697214.707705]: received mask_obs with header seq: 1078 +stamp: + secs: 1735697214 + nsecs: 710818290 +frame_id: '' and grasped_object_mask header seq: 135553 +stamp: + secs: 1735697214 + nsecs: 468456268 +frame_id: "camera_color_optical_frame" +[INFO] [1735697214.932526]: received mask_obs with header seq: 1079 +stamp: + secs: 1735697214 + nsecs: 934296607 +frame_id: '' and grasped_object_mask header seq: 135559 +stamp: + secs: 1735697214 + nsecs: 669400454 +frame_id: "camera_color_optical_frame" +[INFO] [1735697215.132774]: received mask_obs with header seq: 1080 +stamp: + secs: 1735697215 + nsecs: 134771108 +frame_id: '' and grasped_object_mask header seq: 135566 +stamp: + secs: 1735697214 + nsecs: 903945684 +frame_id: "camera_color_optical_frame" +[INFO] [1735697215.334598]: received mask_obs with header seq: 1081 +stamp: + secs: 1735697215 + nsecs: 337303400 +frame_id: '' and grasped_object_mask header seq: 135572 +stamp: + secs: 1735697215 + nsecs: 105079174 +frame_id: "camera_color_optical_frame" +[INFO] [1735697215.536223]: received mask_obs with header seq: 1082 +stamp: + secs: 1735697215 + nsecs: 539706945 +frame_id: '' and grasped_object_mask header seq: 135578 +stamp: + secs: 1735697215 + nsecs: 306299686 +frame_id: "camera_color_optical_frame" +[INFO] [1735697215.733386]: received mask_obs with header seq: 1083 +stamp: + secs: 1735697215 + nsecs: 736877918 +frame_id: '' and grasped_object_mask header seq: 135584 +stamp: + secs: 1735697215 + nsecs: 506624222 +frame_id: "camera_color_optical_frame" +[INFO] [1735697215.933257]: received mask_obs with header seq: 1084 +stamp: + secs: 1735697215 + nsecs: 936567068 +frame_id: '' and grasped_object_mask header seq: 135590 +stamp: + secs: 1735697215 + nsecs: 706473112 +frame_id: "camera_color_optical_frame" +[INFO] [1735697216.130557]: received mask_obs with header seq: 1085 +stamp: + secs: 1735697216 + nsecs: 133783578 +frame_id: '' and grasped_object_mask header seq: 135596 +stamp: + secs: 1735697215 + nsecs: 906321526 +frame_id: "camera_color_optical_frame" +[INFO] [1735697216.333581]: received mask_obs with header seq: 1086 +stamp: + secs: 1735697216 + nsecs: 336902379 +frame_id: '' and grasped_object_mask header seq: 135602 +stamp: + secs: 1735697216 + nsecs: 106170416 +frame_id: "camera_color_optical_frame" +[INFO] [1735697216.532218]: received mask_obs with header seq: 1087 +stamp: + secs: 1735697216 + nsecs: 535585641 +frame_id: '' and grasped_object_mask header seq: 135608 +stamp: + secs: 1735697216 + nsecs: 306019783 +frame_id: "camera_color_optical_frame" +[INFO] [1735697216.732105]: received mask_obs with header seq: 1088 +stamp: + secs: 1735697216 + nsecs: 734688997 +frame_id: '' and grasped_object_mask header seq: 135614 +stamp: + secs: 1735697216 + nsecs: 505862474 +frame_id: "camera_color_optical_frame" +[INFO] [1735697216.927609]: received mask_obs with header seq: 1089 +stamp: + secs: 1735697216 + nsecs: 930941581 +frame_id: '' and grasped_object_mask header seq: 135620 +stamp: + secs: 1735697216 + nsecs: 705701590 +frame_id: "camera_color_optical_frame" +[INFO] [1735697217.120502]: received mask_obs with header seq: 1090 +stamp: + secs: 1735697217 + nsecs: 123743772 +frame_id: '' and grasped_object_mask header seq: 135626 +stamp: + secs: 1735697216 + nsecs: 905542135 +frame_id: "camera_color_optical_frame" +[INFO] [1735697217.308954]: received mask_obs with header seq: 1091 +stamp: + secs: 1735697217 + nsecs: 312174081 +frame_id: '' and grasped_object_mask header seq: 135631 +stamp: + secs: 1735697217 + nsecs: 72074413 +frame_id: "camera_color_optical_frame" +[INFO] [1735697217.507576]: received mask_obs with header seq: 1092 +stamp: + secs: 1735697217 + nsecs: 511017322 +frame_id: '' and grasped_object_mask header seq: 135637 +stamp: + secs: 1735697217 + nsecs: 271914005 +frame_id: "camera_color_optical_frame" +[INFO] [1735697217.708811]: received mask_obs with header seq: 1093 +stamp: + secs: 1735697217 + nsecs: 712276220 +frame_id: '' and grasped_object_mask header seq: 135643 +stamp: + secs: 1735697217 + nsecs: 471755505 +frame_id: "camera_color_optical_frame" +[INFO] [1735697217.907449]: received mask_obs with header seq: 1094 +stamp: + secs: 1735697217 + nsecs: 910845279 +frame_id: '' and grasped_object_mask header seq: 135649 +stamp: + secs: 1735697217 + nsecs: 671597004 +frame_id: "camera_color_optical_frame" +[INFO] [1735697218.109432]: received mask_obs with header seq: 1095 +stamp: + secs: 1735697218 + nsecs: 112859964 +frame_id: '' and grasped_object_mask header seq: 135655 +stamp: + secs: 1735697217 + nsecs: 871438265 +frame_id: "camera_color_optical_frame" +[INFO] [1735697218.310147]: received mask_obs with header seq: 1096 +stamp: + secs: 1735697218 + nsecs: 313057899 +frame_id: '' and grasped_object_mask header seq: 135661 +stamp: + secs: 1735697218 + nsecs: 71279526 +frame_id: "camera_color_optical_frame" +[INFO] [1735697218.507857]: received mask_obs with header seq: 1097 +stamp: + secs: 1735697218 + nsecs: 511317253 +frame_id: '' and grasped_object_mask header seq: 135667 +stamp: + secs: 1735697218 + nsecs: 271121025 +frame_id: "camera_color_optical_frame" +[INFO] [1735697218.707528]: received mask_obs with header seq: 1098 +stamp: + secs: 1735697218 + nsecs: 710954189 +frame_id: '' and grasped_object_mask header seq: 135673 +stamp: + secs: 1735697218 + nsecs: 470961332 +frame_id: "camera_color_optical_frame" +[INFO] [1735697218.908257]: received mask_obs with header seq: 1099 +stamp: + secs: 1735697218 + nsecs: 911690950 +frame_id: '' and grasped_object_mask header seq: 135679 +stamp: + secs: 1735697218 + nsecs: 670800447 +frame_id: "camera_color_optical_frame" +[INFO] [1735697219.109541]: received mask_obs with header seq: 1100 +stamp: + secs: 1735697219 + nsecs: 112994670 +frame_id: '' and grasped_object_mask header seq: 135685 +stamp: + secs: 1735697218 + nsecs: 870640278 +frame_id: "camera_color_optical_frame" +[INFO] [1735697219.309262]: received mask_obs with header seq: 1101 +stamp: + secs: 1735697219 + nsecs: 312531948 +frame_id: '' and grasped_object_mask header seq: 135691 +stamp: + secs: 1735697219 + nsecs: 70479155 +frame_id: "camera_color_optical_frame" +[INFO] [1735697219.509390]: received mask_obs with header seq: 1102 +stamp: + secs: 1735697219 + nsecs: 512697935 +frame_id: '' and grasped_object_mask header seq: 135697 +stamp: + secs: 1735697219 + nsecs: 270317793 +frame_id: "camera_color_optical_frame" +[INFO] [1735697219.709616]: received mask_obs with header seq: 1103 +stamp: + secs: 1735697219 + nsecs: 712984561 +frame_id: '' and grasped_object_mask header seq: 135703 +stamp: + secs: 1735697219 + nsecs: 470155954 +frame_id: "camera_color_optical_frame" +[INFO] [1735697219.911516]: received mask_obs with header seq: 1104 +stamp: + secs: 1735697219 + nsecs: 914982795 +frame_id: '' and grasped_object_mask header seq: 135709 +stamp: + secs: 1735697219 + nsecs: 669993162 +frame_id: "camera_color_optical_frame" +[INFO] [1735697220.110379]: received mask_obs with header seq: 1105 +stamp: + secs: 1735697220 + nsecs: 113334417 +frame_id: '' and grasped_object_mask header seq: 135715 +stamp: + secs: 1735697219 + nsecs: 869830370 +frame_id: "camera_color_optical_frame" +[INFO] [1735697220.309186]: received mask_obs with header seq: 1106 +stamp: + secs: 1735697220 + nsecs: 312493085 +frame_id: '' and grasped_object_mask header seq: 135721 +stamp: + secs: 1735697220 + nsecs: 69668293 +frame_id: "camera_color_optical_frame" +[INFO] [1735697220.508363]: received mask_obs with header seq: 1107 +stamp: + secs: 1735697220 + nsecs: 511761426 +frame_id: '' and grasped_object_mask header seq: 135727 +stamp: + secs: 1735697220 + nsecs: 269505024 +frame_id: "camera_color_optical_frame" +[INFO] [1735697220.709591]: received mask_obs with header seq: 1108 +stamp: + secs: 1735697220 + nsecs: 713019847 +frame_id: '' and grasped_object_mask header seq: 135733 +stamp: + secs: 1735697220 + nsecs: 469340801 +frame_id: "camera_color_optical_frame" +[INFO] [1735697220.907772]: received mask_obs with header seq: 1109 +stamp: + secs: 1735697220 + nsecs: 911161422 +frame_id: '' and grasped_object_mask header seq: 135739 +stamp: + secs: 1735697220 + nsecs: 669175863 +frame_id: "camera_color_optical_frame" +[INFO] [1735697221.109375]: received mask_obs with header seq: 1110 +stamp: + secs: 1735697221 + nsecs: 112801074 +frame_id: '' and grasped_object_mask header seq: 135745 +stamp: + secs: 1735697220 + nsecs: 869010687 +frame_id: "camera_color_optical_frame" +[INFO] [1735697221.308269]: received mask_obs with header seq: 1111 +stamp: + secs: 1735697221 + nsecs: 311645507 +frame_id: '' and grasped_object_mask header seq: 135751 +stamp: + secs: 1735697221 + nsecs: 68845510 +frame_id: "camera_color_optical_frame" +[INFO] [1735697221.507720]: received mask_obs with header seq: 1112 +stamp: + secs: 1735697221 + nsecs: 511139392 +frame_id: '' and grasped_object_mask header seq: 135757 +stamp: + secs: 1735697221 + nsecs: 268680811 +frame_id: "camera_color_optical_frame" +[INFO] [1735697221.709239]: received mask_obs with header seq: 1113 +stamp: + secs: 1735697221 + nsecs: 712463140 +frame_id: '' and grasped_object_mask header seq: 135763 +stamp: + secs: 1735697221 + nsecs: 468520164 +frame_id: "camera_color_optical_frame" +[INFO] [1735697221.927613]: received mask_obs with header seq: 1114 +stamp: + secs: 1735697221 + nsecs: 930740118 +frame_id: '' and grasped_object_mask header seq: 135769 +stamp: + secs: 1735697221 + nsecs: 668365240 +frame_id: "camera_color_optical_frame" +[INFO] [1735697222.124659]: received mask_obs with header seq: 1115 +stamp: + secs: 1735697222 + nsecs: 128109216 +frame_id: '' and grasped_object_mask header seq: 135776 +stamp: + secs: 1735697221 + nsecs: 901518106 +frame_id: "camera_color_optical_frame" +[INFO] [1735697222.322543]: received mask_obs with header seq: 1116 +stamp: + secs: 1735697222 + nsecs: 325857877 +frame_id: '' and grasped_object_mask header seq: 135782 +stamp: + secs: 1735697222 + nsecs: 101363182 +frame_id: "camera_color_optical_frame" +[INFO] [1735697222.509259]: received mask_obs with header seq: 1117 +stamp: + secs: 1735697222 + nsecs: 512446403 +frame_id: '' and grasped_object_mask header seq: 135787 +stamp: + secs: 1735697222 + nsecs: 267900705 +frame_id: "camera_color_optical_frame" +[INFO] [1735697222.709402]: received mask_obs with header seq: 1118 +stamp: + secs: 1735697222 + nsecs: 712823390 +frame_id: '' and grasped_object_mask header seq: 135793 +stamp: + secs: 1735697222 + nsecs: 467744827 +frame_id: "camera_color_optical_frame" +[INFO] [1735697222.909576]: received mask_obs with header seq: 1119 +stamp: + secs: 1735697222 + nsecs: 912964582 +frame_id: '' and grasped_object_mask header seq: 135799 +stamp: + secs: 1735697222 + nsecs: 667586327 +frame_id: "camera_color_optical_frame" +[INFO] [1735697223.110216]: received mask_obs with header seq: 1120 +stamp: + secs: 1735697223 + nsecs: 113618850 +frame_id: '' and grasped_object_mask header seq: 135805 +stamp: + secs: 1735697222 + nsecs: 867428064 +frame_id: "camera_color_optical_frame" +[INFO] [1735697223.308657]: received mask_obs with header seq: 1121 +stamp: + secs: 1735697223 + nsecs: 312067747 +frame_id: '' and grasped_object_mask header seq: 135811 +stamp: + secs: 1735697223 + nsecs: 67269564 +frame_id: "camera_color_optical_frame" +[INFO] [1735697223.507028]: received mask_obs with header seq: 1122 +stamp: + secs: 1735697223 + nsecs: 510353565 +frame_id: '' and grasped_object_mask header seq: 135817 +stamp: + secs: 1735697223 + nsecs: 267111301 +frame_id: "camera_color_optical_frame" +[INFO] [1735697223.726003]: received mask_obs with header seq: 1123 +stamp: + secs: 1735697223 + nsecs: 728315114 +frame_id: '' and grasped_object_mask header seq: 135823 +stamp: + secs: 1735697223 + nsecs: 466948986 +frame_id: "camera_color_optical_frame" +[INFO] [1735697223.925006]: received mask_obs with header seq: 1124 +stamp: + secs: 1735697223 + nsecs: 927361249 +frame_id: '' and grasped_object_mask header seq: 135830 +stamp: + secs: 1735697223 + nsecs: 700085402 +frame_id: "camera_color_optical_frame" +[INFO] [1735697224.123582]: received mask_obs with header seq: 1125 +stamp: + secs: 1735697224 + nsecs: 127025365 +frame_id: '' and grasped_object_mask header seq: 135836 +stamp: + secs: 1735697223 + nsecs: 899915934 +frame_id: "camera_color_optical_frame" +[INFO] [1735697224.326583]: received mask_obs with header seq: 1126 +stamp: + secs: 1735697224 + nsecs: 329703330 +frame_id: '' and grasped_object_mask header seq: 135841 +stamp: + secs: 1735697224 + nsecs: 66441536 +frame_id: "camera_color_optical_frame" +[INFO] [1735697224.530169]: received mask_obs with header seq: 1127 +stamp: + secs: 1735697224 + nsecs: 533507585 +frame_id: '' and grasped_object_mask header seq: 135848 +stamp: + secs: 1735697224 + nsecs: 299576759 +frame_id: "camera_color_optical_frame" +[INFO] [1735697224.729947]: received mask_obs with header seq: 1128 +stamp: + secs: 1735697224 + nsecs: 733380794 +frame_id: '' and grasped_object_mask header seq: 135854 +stamp: + secs: 1735697224 + nsecs: 499409914 +frame_id: "camera_color_optical_frame" +[INFO] [1735697224.932300]: received mask_obs with header seq: 1129 +stamp: + secs: 1735697224 + nsecs: 934970617 +frame_id: '' and grasped_object_mask header seq: 135860 +stamp: + secs: 1735697224 + nsecs: 699246645 +frame_id: "camera_color_optical_frame" +[INFO] [1735697225.129383]: received mask_obs with header seq: 1130 +stamp: + secs: 1735697225 + nsecs: 132838964 +frame_id: '' and grasped_object_mask header seq: 135866 +stamp: + secs: 1735697224 + nsecs: 899082899 +frame_id: "camera_color_optical_frame" +[INFO] [1735697225.325766]: received mask_obs with header seq: 1131 +stamp: + secs: 1735697225 + nsecs: 329232454 +frame_id: '' and grasped_object_mask header seq: 135872 +stamp: + secs: 1735697225 + nsecs: 98919392 +frame_id: "camera_color_optical_frame" +[INFO] [1735697225.524891]: received mask_obs with header seq: 1132 +stamp: + secs: 1735697225 + nsecs: 527987003 +frame_id: '' and grasped_object_mask header seq: 135878 +stamp: + secs: 1735697225 + nsecs: 298755646 +frame_id: "camera_color_optical_frame" +[INFO] [1735697225.720046]: received mask_obs with header seq: 1133 +stamp: + secs: 1735697225 + nsecs: 723305463 +frame_id: '' and grasped_object_mask header seq: 135884 +stamp: + secs: 1735697225 + nsecs: 499357224 +frame_id: "camera_color_optical_frame" +[INFO] [1735697225.918852]: received mask_obs with header seq: 1134 +stamp: + secs: 1735697225 + nsecs: 922125339 +frame_id: '' and grasped_object_mask header seq: 135890 +stamp: + secs: 1735697225 + nsecs: 701543570 +frame_id: "camera_color_optical_frame" +[INFO] [1735697226.116925]: received mask_obs with header seq: 1135 +stamp: + secs: 1735697226 + nsecs: 120400667 +frame_id: '' and grasped_object_mask header seq: 135895 +stamp: + secs: 1735697225 + nsecs: 870098591 +frame_id: "camera_color_optical_frame" +[INFO] [1735697226.309768]: received mask_obs with header seq: 1136 +stamp: + secs: 1735697226 + nsecs: 312946081 +frame_id: '' and grasped_object_mask header seq: 135901 +stamp: + secs: 1735697226 + nsecs: 72445869 +frame_id: "camera_color_optical_frame" +[INFO] [1735697226.511260]: received mask_obs with header seq: 1137 +stamp: + secs: 1735697226 + nsecs: 514536380 +frame_id: '' and grasped_object_mask header seq: 135907 +stamp: + secs: 1735697226 + nsecs: 274881840 +frame_id: "camera_color_optical_frame" +[INFO] [1735697226.720035]: received mask_obs with header seq: 1138 +stamp: + secs: 1735697226 + nsecs: 723520278 +frame_id: '' and grasped_object_mask header seq: 135913 +stamp: + secs: 1735697226 + nsecs: 476915121 +frame_id: "camera_color_optical_frame" +[INFO] [1735697226.925003]: received mask_obs with header seq: 1139 +stamp: + secs: 1735697226 + nsecs: 928185701 +frame_id: '' and grasped_object_mask header seq: 135920 +stamp: + secs: 1735697226 + nsecs: 709797144 +frame_id: "camera_color_optical_frame" +[INFO] [1735697227.124445]: received mask_obs with header seq: 1140 +stamp: + secs: 1735697227 + nsecs: 127134799 +frame_id: '' and grasped_object_mask header seq: 135926 +stamp: + secs: 1735697226 + nsecs: 909399033 +frame_id: "camera_color_optical_frame" +[INFO] [1735697227.324403]: received mask_obs with header seq: 1141 +stamp: + secs: 1735697227 + nsecs: 327038049 +frame_id: '' and grasped_object_mask header seq: 135932 +stamp: + secs: 1735697227 + nsecs: 108988047 +frame_id: "camera_color_optical_frame" +[INFO] [1735697227.520153]: received mask_obs with header seq: 1142 +stamp: + secs: 1735697227 + nsecs: 522778272 +frame_id: '' and grasped_object_mask header seq: 135938 +stamp: + secs: 1735697227 + nsecs: 308565617 +frame_id: "camera_color_optical_frame" +[INFO] [1735697227.709862]: received mask_obs with header seq: 1143 +stamp: + secs: 1735697227 + nsecs: 713149785 +frame_id: '' and grasped_object_mask header seq: 135944 +stamp: + secs: 1735697227 + nsecs: 508148193 +frame_id: "camera_color_optical_frame" +[INFO] [1735697227.917915]: received mask_obs with header seq: 1144 +stamp: + secs: 1735697227 + nsecs: 921331405 +frame_id: '' and grasped_object_mask header seq: 135950 +stamp: + secs: 1735697227 + nsecs: 707752466 +frame_id: "camera_color_optical_frame" +[INFO] [1735697228.116478]: received mask_obs with header seq: 1145 +stamp: + secs: 1735697228 + nsecs: 119673490 +frame_id: '' and grasped_object_mask header seq: 135956 +stamp: + secs: 1735697227 + nsecs: 907345057 +frame_id: "camera_color_optical_frame" +[INFO] [1735697228.312705]: received mask_obs with header seq: 1146 +stamp: + secs: 1735697228 + nsecs: 315785169 +frame_id: '' and grasped_object_mask header seq: 135962 +stamp: + secs: 1735697228 + nsecs: 106926203 +frame_id: "camera_color_optical_frame" +[INFO] [1735697228.517066]: received mask_obs with header seq: 1147 +stamp: + secs: 1735697228 + nsecs: 520432949 +frame_id: '' and grasped_object_mask header seq: 135968 +stamp: + secs: 1735697228 + nsecs: 306494474 +frame_id: "camera_color_optical_frame" +[INFO] [1735697228.717351]: received mask_obs with header seq: 1148 +stamp: + secs: 1735697228 + nsecs: 720816612 +frame_id: '' and grasped_object_mask header seq: 135974 +stamp: + secs: 1735697228 + nsecs: 506070375 +frame_id: "camera_color_optical_frame" +[INFO] [1735697228.919253]: received mask_obs with header seq: 1149 +stamp: + secs: 1735697228 + nsecs: 922370910 +frame_id: '' and grasped_object_mask header seq: 135980 +stamp: + secs: 1735697228 + nsecs: 705673456 +frame_id: "camera_color_optical_frame" +[INFO] [1735697229.120383]: received mask_obs with header seq: 1150 +stamp: + secs: 1735697229 + nsecs: 123504161 +frame_id: '' and grasped_object_mask header seq: 135986 +stamp: + secs: 1735697228 + nsecs: 905264378 +frame_id: "camera_color_optical_frame" +[INFO] [1735697229.324466]: received mask_obs with header seq: 1151 +stamp: + secs: 1735697229 + nsecs: 327971935 +frame_id: '' and grasped_object_mask header seq: 135992 +stamp: + secs: 1735697229 + nsecs: 104843855 +frame_id: "camera_color_optical_frame" +[INFO] [1735697229.521717]: received mask_obs with header seq: 1152 +stamp: + secs: 1735697229 + nsecs: 525211095 +frame_id: '' and grasped_object_mask header seq: 135998 +stamp: + secs: 1735697229 + nsecs: 304412127 +frame_id: "camera_color_optical_frame" +[INFO] [1735697229.722878]: received mask_obs with header seq: 1153 +stamp: + secs: 1735697229 + nsecs: 726367950 +frame_id: '' and grasped_object_mask header seq: 136004 +stamp: + secs: 1735697229 + nsecs: 503991127 +frame_id: "camera_color_optical_frame" +[INFO] [1735697229.918285]: received mask_obs with header seq: 1154 +stamp: + secs: 1735697229 + nsecs: 921008348 +frame_id: '' and grasped_object_mask header seq: 136010 +stamp: + secs: 1735697229 + nsecs: 703606129 +frame_id: "camera_color_optical_frame" +[INFO] [1735697230.116402]: received mask_obs with header seq: 1155 +stamp: + secs: 1735697230 + nsecs: 119146823 +frame_id: '' and grasped_object_mask header seq: 136016 +stamp: + secs: 1735697229 + nsecs: 903209925 +frame_id: "camera_color_optical_frame" +[INFO] [1735697230.315961]: received mask_obs with header seq: 1156 +stamp: + secs: 1735697230 + nsecs: 319414377 +frame_id: '' and grasped_object_mask header seq: 136022 +stamp: + secs: 1735697230 + nsecs: 102802277 +frame_id: "camera_color_optical_frame" +[INFO] [1735697230.516309]: received mask_obs with header seq: 1157 +stamp: + secs: 1735697230 + nsecs: 518768310 +frame_id: '' and grasped_object_mask header seq: 136028 +stamp: + secs: 1735697230 + nsecs: 302382946 +frame_id: "camera_color_optical_frame" +[INFO] [1735697230.723173]: received mask_obs with header seq: 1158 +stamp: + secs: 1735697230 + nsecs: 726666927 +frame_id: '' and grasped_object_mask header seq: 136034 +stamp: + secs: 1735697230 + nsecs: 501971245 +frame_id: "camera_color_optical_frame" +[INFO] [1735697230.922083]: received mask_obs with header seq: 1159 +stamp: + secs: 1735697230 + nsecs: 925438880 +frame_id: '' and grasped_object_mask header seq: 136040 +stamp: + secs: 1735697230 + nsecs: 701581478 +frame_id: "camera_color_optical_frame" +[INFO] [1735697231.120565]: received mask_obs with header seq: 1160 +stamp: + secs: 1735697231 + nsecs: 123285055 +frame_id: '' and grasped_object_mask header seq: 136046 +stamp: + secs: 1735697230 + nsecs: 901180506 +frame_id: "camera_color_optical_frame" +[INFO] [1735697231.320397]: received mask_obs with header seq: 1161 +stamp: + secs: 1735697231 + nsecs: 323236942 +frame_id: '' and grasped_object_mask header seq: 136052 +stamp: + secs: 1735697231 + nsecs: 100767612 +frame_id: "camera_color_optical_frame" +[INFO] [1735697231.516133]: received mask_obs with header seq: 1162 +stamp: + secs: 1735697231 + nsecs: 518776416 +frame_id: '' and grasped_object_mask header seq: 136058 +stamp: + secs: 1735697231 + nsecs: 300343275 +frame_id: "camera_color_optical_frame" +[INFO] [1735697231.715943]: received mask_obs with header seq: 1163 +stamp: + secs: 1735697231 + nsecs: 718523979 +frame_id: '' and grasped_object_mask header seq: 136064 +stamp: + secs: 1735697231 + nsecs: 499926329 +frame_id: "camera_color_optical_frame" +[INFO] [1735697231.915445]: received mask_obs with header seq: 1164 +stamp: + secs: 1735697231 + nsecs: 918928623 +frame_id: '' and grasped_object_mask header seq: 136070 +stamp: + secs: 1735697231 + nsecs: 699536562 +frame_id: "camera_color_optical_frame" +[INFO] [1735697232.117257]: received mask_obs with header seq: 1165 +stamp: + secs: 1735697232 + nsecs: 120705127 +frame_id: '' and grasped_object_mask header seq: 136076 +stamp: + secs: 1735697231 + nsecs: 899134159 +frame_id: "camera_color_optical_frame" +[INFO] [1735697232.313864]: received mask_obs with header seq: 1166 +stamp: + secs: 1735697232 + nsecs: 317239761 +frame_id: '' and grasped_object_mask header seq: 136082 +stamp: + secs: 1735697232 + nsecs: 98720312 +frame_id: "camera_color_optical_frame" +[INFO] [1735697232.514584]: received mask_obs with header seq: 1167 +stamp: + secs: 1735697232 + nsecs: 517709732 +frame_id: '' and grasped_object_mask header seq: 136088 +stamp: + secs: 1735697232 + nsecs: 298294544 +frame_id: "camera_color_optical_frame" +[INFO] [1735697232.716337]: received mask_obs with header seq: 1168 +stamp: + secs: 1735697232 + nsecs: 719823598 +frame_id: '' and grasped_object_mask header seq: 136094 +stamp: + secs: 1735697232 + nsecs: 497876644 +frame_id: "camera_color_optical_frame" +[INFO] [1735697232.915424]: received mask_obs with header seq: 1169 +stamp: + secs: 1735697232 + nsecs: 918560504 +frame_id: '' and grasped_object_mask header seq: 136100 +stamp: + secs: 1735697232 + nsecs: 697486401 +frame_id: "camera_color_optical_frame" +[INFO] [1735697233.115796]: received mask_obs with header seq: 1170 +stamp: + secs: 1735697233 + nsecs: 118995666 +frame_id: '' and grasped_object_mask header seq: 136106 +stamp: + secs: 1735697232 + nsecs: 897084951 +frame_id: "camera_color_optical_frame" +[INFO] [1735697233.316198]: received mask_obs with header seq: 1171 +stamp: + secs: 1735697233 + nsecs: 319573163 +frame_id: '' and grasped_object_mask header seq: 136112 +stamp: + secs: 1735697233 + nsecs: 96671104 +frame_id: "camera_color_optical_frame" +[INFO] [1735697233.515606]: received mask_obs with header seq: 1172 +stamp: + secs: 1735697233 + nsecs: 518557786 +frame_id: '' and grasped_object_mask header seq: 136118 +stamp: + secs: 1735697233 + nsecs: 296245337 +frame_id: "camera_color_optical_frame" +[INFO] [1735697233.714072]: received mask_obs with header seq: 1173 +stamp: + secs: 1735697233 + nsecs: 717520713 +frame_id: '' and grasped_object_mask header seq: 136124 +stamp: + secs: 1735697233 + nsecs: 495828867 +frame_id: "camera_color_optical_frame" +[INFO] [1735697233.914943]: received mask_obs with header seq: 1174 +stamp: + secs: 1735697233 + nsecs: 918281555 +frame_id: '' and grasped_object_mask header seq: 136130 +stamp: + secs: 1735697233 + nsecs: 695443630 +frame_id: "camera_color_optical_frame" +[INFO] [1735697234.121911]: received mask_obs with header seq: 1175 +stamp: + secs: 1735697234 + nsecs: 125255346 +frame_id: '' and grasped_object_mask header seq: 136136 +stamp: + secs: 1735697233 + nsecs: 895046711 +frame_id: "camera_color_optical_frame" +[INFO] [1735697234.327575]: received mask_obs with header seq: 1176 +stamp: + secs: 1735697234 + nsecs: 331048965 +frame_id: '' and grasped_object_mask header seq: 136142 +stamp: + secs: 1735697234 + nsecs: 94638109 +frame_id: "camera_color_optical_frame" +[INFO] [1735697234.523771]: received mask_obs with header seq: 1177 +stamp: + secs: 1735697234 + nsecs: 526843786 +frame_id: '' and grasped_object_mask header seq: 136148 +stamp: + secs: 1735697234 + nsecs: 294219017 +frame_id: "camera_color_optical_frame" +[INFO] [1735697234.718462]: received mask_obs with header seq: 1178 +stamp: + secs: 1735697234 + nsecs: 721503257 +frame_id: '' and grasped_object_mask header seq: 136154 +stamp: + secs: 1735697234 + nsecs: 493805408 +frame_id: "camera_color_optical_frame" +[INFO] [1735697234.916588]: received mask_obs with header seq: 1179 +stamp: + secs: 1735697234 + nsecs: 919911384 +frame_id: '' and grasped_object_mask header seq: 136160 +stamp: + secs: 1735697234 + nsecs: 693416357 +frame_id: "camera_color_optical_frame" +[INFO] [1735697235.118360]: received mask_obs with header seq: 1180 +stamp: + secs: 1735697235 + nsecs: 121781349 +frame_id: '' and grasped_object_mask header seq: 136166 +stamp: + secs: 1735697234 + nsecs: 893015146 +frame_id: "camera_color_optical_frame" +[INFO] [1735697235.313529]: received mask_obs with header seq: 1181 +stamp: + secs: 1735697235 + nsecs: 316840648 +frame_id: '' and grasped_object_mask header seq: 136172 +stamp: + secs: 1735697235 + nsecs: 92602491 +frame_id: "camera_color_optical_frame" +[INFO] [1735697235.512091]: received mask_obs with header seq: 1182 +stamp: + secs: 1735697235 + nsecs: 515552520 +frame_id: '' and grasped_object_mask header seq: 136178 +stamp: + secs: 1735697235 + nsecs: 292177916 +frame_id: "camera_color_optical_frame" +[INFO] [1735697235.719333]: received mask_obs with header seq: 1183 +stamp: + secs: 1735697235 + nsecs: 722762346 +frame_id: '' and grasped_object_mask header seq: 136184 +stamp: + secs: 1735697235 + nsecs: 491760969 +frame_id: "camera_color_optical_frame" +[INFO] [1735697235.917182]: received mask_obs with header seq: 1184 +stamp: + secs: 1735697235 + nsecs: 920670509 +frame_id: '' and grasped_object_mask header seq: 136190 +stamp: + secs: 1735697235 + nsecs: 691367626 +frame_id: "camera_color_optical_frame" +[INFO] [1735697236.116931]: received mask_obs with header seq: 1185 +stamp: + secs: 1735697236 + nsecs: 120329618 +frame_id: '' and grasped_object_mask header seq: 136196 +stamp: + secs: 1735697235 + nsecs: 890962362 +frame_id: "camera_color_optical_frame" +[INFO] [1735697236.314756]: received mask_obs with header seq: 1186 +stamp: + secs: 1735697236 + nsecs: 317959070 +frame_id: '' and grasped_object_mask header seq: 136202 +stamp: + secs: 1735697236 + nsecs: 90545177 +frame_id: "camera_color_optical_frame" +[INFO] [1735697236.517906]: received mask_obs with header seq: 1187 +stamp: + secs: 1735697236 + nsecs: 521208286 +frame_id: '' and grasped_object_mask header seq: 136208 +stamp: + secs: 1735697236 + nsecs: 290116072 +frame_id: "camera_color_optical_frame" +[INFO] [1735697236.725254]: received mask_obs with header seq: 1188 +stamp: + secs: 1735697236 + nsecs: 728735446 +frame_id: '' and grasped_object_mask header seq: 136214 +stamp: + secs: 1735697236 + nsecs: 489694595 +frame_id: "camera_color_optical_frame" +[INFO] [1735697236.922326]: received mask_obs with header seq: 1189 +stamp: + secs: 1735697236 + nsecs: 925775289 +frame_id: '' and grasped_object_mask header seq: 136220 +stamp: + secs: 1735697236 + nsecs: 689301252 +frame_id: "camera_color_optical_frame" +[INFO] [1735697237.117040]: received mask_obs with header seq: 1190 +stamp: + secs: 1735697237 + nsecs: 120565176 +frame_id: '' and grasped_object_mask header seq: 136226 +stamp: + secs: 1735697236 + nsecs: 888895035 +frame_id: "camera_color_optical_frame" +[INFO] [1735697237.313333]: received mask_obs with header seq: 1191 +stamp: + secs: 1735697237 + nsecs: 316771745 +frame_id: '' and grasped_object_mask header seq: 136232 +stamp: + secs: 1735697237 + nsecs: 88476658 +frame_id: "camera_color_optical_frame" +[INFO] [1735697237.512517]: received mask_obs with header seq: 1192 +stamp: + secs: 1735697237 + nsecs: 515973567 +frame_id: '' and grasped_object_mask header seq: 136238 +stamp: + secs: 1735697237 + nsecs: 288046360 +frame_id: "camera_color_optical_frame" +[INFO] [1735697237.712038]: received mask_obs with header seq: 1193 +stamp: + secs: 1735697237 + nsecs: 715522289 +frame_id: '' and grasped_object_mask header seq: 136244 +stamp: + secs: 1735697237 + nsecs: 487626076 +frame_id: "camera_color_optical_frame" +[INFO] [1735697237.915880]: received mask_obs with header seq: 1194 +stamp: + secs: 1735697237 + nsecs: 919351339 +frame_id: '' and grasped_object_mask header seq: 136250 +stamp: + secs: 1735697237 + nsecs: 687237978 +frame_id: "camera_color_optical_frame" +[INFO] [1735697238.116964]: received mask_obs with header seq: 1195 +stamp: + secs: 1735697238 + nsecs: 120110988 +frame_id: '' and grasped_object_mask header seq: 136256 +stamp: + secs: 1735697237 + nsecs: 886839628 +frame_id: "camera_color_optical_frame" +[INFO] [1735697238.312885]: received mask_obs with header seq: 1196 +stamp: + secs: 1735697238 + nsecs: 316371202 +frame_id: '' and grasped_object_mask header seq: 136262 +stamp: + secs: 1735697238 + nsecs: 86428404 +frame_id: "camera_color_optical_frame" +[INFO] [1735697238.514093]: received mask_obs with header seq: 1197 +stamp: + secs: 1735697238 + nsecs: 517579317 +frame_id: '' and grasped_object_mask header seq: 136268 +stamp: + secs: 1735697238 + nsecs: 286005974 +frame_id: "camera_color_optical_frame" +[INFO] [1735697238.557500]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735697238.715171]: received mask_obs with header seq: 1198 +stamp: + secs: 1735697238 + nsecs: 718674898 +frame_id: '' and grasped_object_mask header seq: 136274 +stamp: + secs: 1735697238 + nsecs: 485591173 +frame_id: "camera_color_optical_frame" +[INFO] [1735697239.357374]: Stored episode 5. +[INFO] [1735697239.363396]: Saving video... +[INFO] [1735697239.895876]: Video saved! +[INFO] [1735697239.956113]: resetting environment +[INFO] [1735697239.976052]: cleared current plan +[INFO] [1735697239.976530]: moving to home +[WARN] [1735697239.890642]: Plan exhausted +[WARN] [1735697239.931569]: Plan exhausted +[WARN] [1735697239.970929]: Plan exhausted +[INFO] [1735697243.580497]: reached home +[INFO] [1735697243.601912]: reset action history +[INFO] [1735697246.451188]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7707a85315efd0323df1482ee16a206249a22b76 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T02:03:23.773340", + "startedAt": "2025-01-01T02:03:23.425372", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "284407f87cd673789a10dbd8e4d4f41dcad30ded" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.004875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.995, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 495.41462326049805 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5e465813df46b042f2408edd8c984c516fc3fd24 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "e739aa3a1064d7a115c648ec192bb1b5b34388ac6d31936a7ef6fd1ca52ae4ea", "size": 678402, "path": "media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4"}, "global_step": 533, "_timestamp": 1735697248.7542074, "_runtime": 245.3166904449463, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "53a45cc58516bfa8280677b9bd2af88e662726e4efdf25b8e31cbbf385486eed", "size": 594378, "path": "media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4"}, "eval/num_success": 3.0, "episode": 5.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "fe1c0520eab44850309611a44e67873cd28908f9005d185df5b339ad1874bcd7", "size": 1157154, "path": "media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a0e47918912079386856849116782c06f1a9ea624d9c21c42d468e6a4df4f115", "size": 1157639, "path": "media/videos/eval/3_eval_3_a0e47918912079386856.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c43d65e2ea74e852f815b6bdd0071163c9bd16ea1f6cadc977d529abe360e33d", "size": 1171802, "path": "media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://6m8a5esai9gytvadz7qpwtg0p69swj61pon97cegh1kw1getxbj319zruncuzk355dv8w2lzfhbd63mwm0wjbu5phgs2j96tlk82ax9ieh2xug2rx05j1abncvpfxt2x/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kb7z1wlsukm0vbzeoytc3npe00ryepww7izmizsads88j8adyuss4ouo75ps6mlfte1s047mxqh4zm7x4en2lininmqtkcwgw1wledwavn7fqlcubs7wd99qbw3e95gn:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 3, "total_num_episodes": 5, "total_success_rate": 0.6, "total_success_rate_lb_0.01": 0.12214982517170042, "total_success_rate_ub_0.01": 0.9567054509203167, "total_success_rate_lb_0.025": 0.2083774209605767, "total_success_rate_ub_0.025": 0.9455957635345953, "total_success_rate_lb_0.05": 0.25968933110274955, "total_success_rate_ub_0.05": 0.8756684585654417, "total_success_rate_lb_0.1": 0.2662367970218884, "total_success_rate_ub_0.1": 0.8798631577924128, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "9152843e0aab0648a7f1ea24e25ac10946c65c2d6ff9cfa2cebf300b9e72407e", "size": 505, "artifact_path": "wandb-client-artifact://z1feg9p4hxsotddhxg71l16j03x4z3cy9ed54nq67vbvmkqvywk68sl837eowkjbxu6fczf6ixa20pmrsx0w59unh6b82t61wk9mpih39h1qtgh220mgwybb2n08ddyl/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://1pvazlnshx9szk2ra8pu2qoypijivvl71sohjr9tbojt3w7ux8m9onn5q877tsffbzwm5k5uyv482c5qxlziggp9ujn9e76hqt1wcs3culqqvdc5mwha3yd3aj2eeclo:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 259}} \ No newline at end of file diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug-internal.log b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ffb40e62e27cf598ed7ad2d484ab9ecfd7d6037a --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug-internal.log @@ -0,0 +1,807 @@ +2024-12-31 21:03:23,435 INFO StreamThr :1740211 [internal.py:wandb_internal():85] W&B internal server running at pid: 1740211, started at: 2024-12-31 21:03:23.435568 +2024-12-31 21:03:23,436 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:03:23,449 INFO WriterThread:1740211 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb +2024-12-31 21:03:23,450 DEBUG SenderThread:1740211 [sender.py:send():379] send: header +2024-12-31 21:03:23,459 DEBUG SenderThread:1740211 [sender.py:send():379] send: run +2024-12-31 21:03:23,694 INFO SenderThread:1740211 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files +2024-12-31 21:03:23,694 INFO SenderThread:1740211 [sender.py:_start_run_threads():1188] run started: nkdxms1x with start time 1735697003.437517 +2024-12-31 21:03:23,711 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:03:23,711 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:03:23,755 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:03:23,760 DEBUG HandlerThread:1740211 [system_info.py:__init__():26] System info init +2024-12-31 21:03:23,760 DEBUG HandlerThread:1740211 [system_info.py:__init__():41] System info init done +2024-12-31 21:03:23,760 INFO HandlerThread:1740211 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:03:23,760 INFO SystemMonitor:1740211 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:03:23,761 INFO HandlerThread:1740211 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:03:23,761 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:03:23,762 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:03:23,762 INFO SystemMonitor:1740211 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:03:23,773 DEBUG HandlerThread:1740211 [system_info.py:probe():152] Probing system +2024-12-31 21:03:23,774 DEBUG HandlerThread:1740211 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:probe():200] Probing system done +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:03:23.773340', 'startedAt': '2025-01-01T02:03:23.425372', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.004875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.995, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 495.41462326049805}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:03:23,779 INFO HandlerThread:1740211 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:03:23,779 INFO HandlerThread:1740211 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:03:23,779 DEBUG HandlerThread:1740211 [system_info.py:_save_code():44] Saving code +2024-12-31 21:03:23,780 DEBUG HandlerThread:1740211 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:03:23,781 DEBUG HandlerThread:1740211 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:03:23,842 DEBUG HandlerThread:1740211 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:03:23,842 INFO HandlerThread:1740211 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:03:23,845 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:03:23,845 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:03:23,846 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:03:23,846 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:03:24,088 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:03:24,088 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:03:24,089 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/q79k8vpc-code/FISH/eval_robot.py +2024-12-31 21:03:24,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:24,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:24,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:24,091 INFO HandlerThread:1740211 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/tb" + save: true +} +control { + relay_id: "ee70b23f69f642a48cfae1d64e8e8ef4" +} +_info { + stream_id: "nkdxms1x" +} + +2024-12-31 21:03:24,091 DEBUG HandlerThread:1740211 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:03:24,094 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/e8mji6dz-wandb-metadata.json +2024-12-31 21:03:24,107 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ti5udqhq-diff.patch +2024-12-31 21:03:24,159 DEBUG SenderThread:1740211 [sender.py:send():379] send: telemetry +2024-12-31 21:03:24,159 DEBUG SenderThread:1740211 [sender.py:send():379] send: tbrecord +2024-12-31 21:03:24,160 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:03:24,160 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 with policy live +2024-12-31 21:03:24,345 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/xguvxaej-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch +2024-12-31 21:03:24,696 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH +2024-12-31 21:03:24,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code +2024-12-31 21:03:25,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:26,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:26,695 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:27,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:28,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:29,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:29,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:29,760 DEBUG SenderThread:1740211 [sender.py:send():379] send: config +2024-12-31 21:03:30,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:31,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:32,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:32,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:33,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:34,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:34,495 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:34,697 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:35,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:36,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:36,698 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:37,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:38,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:38,699 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:39,088 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:39,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:39,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:40,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:40,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:41,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:42,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:43,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:44,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:45,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:45,988 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:46,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:46,701 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:47,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:48,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:48,701 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:49,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:50,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:50,702 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:51,015 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:51,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:52,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:52,704 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:53,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:54,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:03:54,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:03:54,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:54,704 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:55,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:56,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:56,178 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:03:56,705 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:03:56,705 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:57,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:58,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:03:58,706 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:03:59,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:00,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:00,706 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:01,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:01,508 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:02,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:02,707 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:03,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:04,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:04,559 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:04:04,559 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 with policy now +2024-12-31 21:04:04,561 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:04:04,562 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:04:04,562 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:04:04,562 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:04:04,707 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media +2024-12-31 21:04:04,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos +2024-12-31 21:04:04,797 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/uj3y4dh9-media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:05,052 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/cgpichjt-media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:04:05,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:06,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:06,573 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:06,708 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:07,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:08,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:09,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:09,091 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:09,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:10,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:10,709 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:11,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:12,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:12,533 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:12,710 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:13,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:14,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:15,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:16,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:17,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:18,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:18,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:19,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:20,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:21,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:22,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:23,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:23,762 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:04:23,763 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:04:23,764 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:24,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:24,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:24,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:25,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:26,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:27,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:28,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:29,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:29,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:30,096 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:31,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:32,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:33,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:34,091 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:34,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:35,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:36,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:36,718 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:04:37,005 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/xgxz2wki-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:04:37,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:38,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:38,719 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:39,089 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:39,089 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:39,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:39,237 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:40,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:41,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:42,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:42,720 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:43,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:44,109 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:44,407 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:44,721 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:45,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:46,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:46,722 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:47,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:48,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:48,723 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:49,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:49,414 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:50,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:50,723 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:51,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:52,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:52,724 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:53,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:53,763 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:04:54,092 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:04:54,092 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:04:54,118 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:54,327 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:04:54,327 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 with policy now +2024-12-31 21:04:54,331 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:04:54,332 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:04:54,332 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:04:54,332 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:04:54,567 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/m9tsz8o4-media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:54,725 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:04:54,725 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:54,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:54,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:04:54,973 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/apzt3djp-media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:04:55,117 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:55,354 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:04:56,115 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:56,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:57,116 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:58,120 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:04:58,726 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:04:59,126 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:00,127 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:00,627 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:00,727 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:01,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:02,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:02,728 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:03,125 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:04,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:05,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:06,093 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:06,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:07,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:08,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:09,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:09,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:09,133 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:10,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:11,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:11,211 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:12,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:13,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:14,128 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:15,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:16,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:17,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:17,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:18,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:19,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:19,501 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:19,501 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:19,502 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:19,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:19,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:05:20,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:20,734 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:21,129 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:22,130 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:22,269 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:22,735 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:23,144 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:23,764 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:05:24,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:24,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:24,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:24,736 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:25,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:26,135 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:26,737 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:27,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:27,441 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:28,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:28,738 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:29,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:30,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:30,738 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:31,136 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:32,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:32,448 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:32,739 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:33,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:34,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:34,740 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:35,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:36,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:36,741 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:37,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:37,469 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:38,137 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:38,741 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:39,090 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:39,090 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:39,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:40,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:40,742 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:41,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:42,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:42,642 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:42,743 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:43,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:44,138 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:44,744 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:45,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:46,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:46,744 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:47,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:47,650 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:48,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:48,745 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:49,139 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:50,142 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:50,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:50,924 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:05:50,924 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 with policy now +2024-12-31 21:05:50,924 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:05:50,925 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:50,925 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:50,925 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:51,143 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:51,174 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ueb0i9jh-media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:51,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:51,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:51,747 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:05:52,054 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/1ppz161i-media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:05:52,141 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:52,746 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:52,959 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:53,146 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:53,764 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:05:54,102 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:05:54,102 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:05:54,149 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:55,147 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:56,145 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:56,748 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:57,145 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:58,146 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:58,393 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:05:58,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:05:59,148 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:05:59,329 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:05:59,330 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:05:59,330 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:05:59,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:05:59,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:05:59,980 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/iaw5gjma-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:06:00,149 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:00,749 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:01,151 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:02,151 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:03,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:03,471 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:04,165 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:04,751 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:05,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:06,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:06,752 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:07,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:08,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:08,502 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:08,752 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:09,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:09,095 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:09,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:10,166 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:10,753 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:11,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:12,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:12,754 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:13,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:13,697 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:14,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:14,755 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:15,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:16,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:16,755 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:17,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:18,167 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:18,698 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:18,756 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:19,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:20,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:20,757 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:21,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:22,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:22,758 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:23,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:23,702 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:23,765 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:06:24,094 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:24,095 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:24,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:24,758 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:25,168 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:26,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:26,759 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:27,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:28,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:28,760 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:28,906 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:29,169 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:30,173 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:30,760 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:31,152 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:06:31,153 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a0e47918912079386856.mp4 with policy now +2024-12-31 21:06:31,156 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:06:31,156 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:06:31,156 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:06:31,157 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:06:31,171 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:31,412 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/tjltem0s-media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:31,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:06:32,054 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/a1eck4xq-media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:06:32,185 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:32,761 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:33,180 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:34,107 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:34,178 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:35,180 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:36,185 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:36,763 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:37,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:38,181 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:38,763 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:39,098 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:39,098 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:39,189 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:39,210 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:40,183 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:41,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:42,188 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:43,186 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:44,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:45,106 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:45,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:46,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:47,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:47,766 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:06:48,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:48,388 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:06:48,388 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:06:48,389 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:06:48,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:06:48,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:49,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:50,122 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:50,197 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:50,767 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:51,196 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:52,190 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:52,768 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:53,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:53,769 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:06:54,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:06:54,096 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:06:54,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:54,770 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:55,133 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:06:55,191 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:56,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:56,770 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:57,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:58,192 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:06:58,771 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:06:59,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:00,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:00,309 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:00,772 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:01,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:02,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:02,773 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:03,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:04,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:04,773 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:05,193 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:05,326 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:06,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:06,774 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:07,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:08,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:08,775 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:09,095 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:09,096 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:09,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:10,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:10,516 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:10,776 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:11,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:12,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:12,777 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:13,194 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:14,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:14,777 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:15,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:15,719 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:16,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:16,778 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:17,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:18,195 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:18,779 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:19,201 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:19,953 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:19,953 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 with policy now +2024-12-31 21:07:19,956 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:19,956 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:19,956 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:19,957 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:20,199 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:20,214 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/n4g10zrq-media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:20,780 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval +2024-12-31 21:07:20,976 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:21,048 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/u21i1vsf-media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:21,201 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:22,220 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:22,781 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:23,206 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:23,769 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:07:24,113 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:24,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:24,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:24,781 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:25,223 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:26,231 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:26,249 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:27,207 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:28,208 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:28,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:28,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,021 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/a07y7rzg-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:29,245 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:07:29,245 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:07:29,477 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:29,699 INFO wandb-upload_1:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo_cu9z3a +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:29,783 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media +2024-12-31 21:07:29,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval +2024-12-31 21:07:29,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table +2024-12-31 21:07:29,830 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:07:29,842 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:07:30,102 INFO SenderThread:1740211 [sender.py:send_request_log_artifact():1525] logged artifact run-nkdxms1x-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTA3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2NjU0NQ==', 'latestArtifact': None}} +2024-12-31 21:07:30,104 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:30,104 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2024-12-31 21:07:30,105 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,105 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,106 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,107 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,107 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,108 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,108 DEBUG SenderThread:1740211 [sender.py:send():379] send: telemetry +2024-12-31 21:07:30,108 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,108 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,109 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,109 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,109 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,109 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,110 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,111 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,111 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,112 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,112 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,112 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,112 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,113 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,113 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:30,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:07:30,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:30,277 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/ly18bboo-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:30,497 INFO wandb-upload_1:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvrm_yjbf +2024-12-31 21:07:30,526 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/bhwrc5ru-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:30,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval +2024-12-31 21:07:30,850 INFO SenderThread:1740211 [sender.py:send_request_log_artifact():1525] logged artifact run-nkdxms1x-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTA4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2NjU0Nw==', 'latestArtifact': None}} +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send():379] send: files +2024-12-31 21:07:30,851 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json with policy now +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:30,851 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:30,851 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:31,032 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/089q6my2-media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:31,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:31,303 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/leshttye-media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:31,784 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:31,852 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:32,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:33,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:34,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:35,203 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:36,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:37,118 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:37,219 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:37,786 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:07:38,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:39,097 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:07:39,097 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:07:39,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:40,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:41,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:42,204 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:43,069 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:43,110 DEBUG SenderThread:1740211 [sender.py:send():379] send: exit +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():588] handling runtime: 259 +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:43,110 INFO SenderThread:1740211 [sender.py:send_exit():594] send defer +2024-12-31 21:07:43,110 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,110 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:07:43,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:07:43,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:07:43,111 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:07:43,111 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:07:43,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:07:43,111 INFO HandlerThread:1740211 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:07:43,111 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:07:43,112 DEBUG SystemMonitor:1740211 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:07:43,112 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:07:43,112 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:07:43,163 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:07:43,164 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:07:43,164 INFO HandlerThread:1740211 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:07:43,164 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,164 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:07:43,164 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:07:43,164 DEBUG SenderThread:1740211 [sender.py:send():379] send: stats +2024-12-31 21:07:43,164 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,165 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:07:43,165 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:43,165 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:07:43,165 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:07:43,165 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:43,165 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:07:43,789 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:44,789 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:49,443 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,444 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:07:49,444 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:49,444 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,444 DEBUG SenderThread:1740211 [sender.py:send():379] send: history +2024-12-31 21:07:49,444 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:07:49,445 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:49,446 DEBUG SenderThread:1740211 [sender.py:send():379] send: summary +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:07:49,446 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:07:49,446 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:07:49,446 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,446 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:07:49,447 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:07:49,447 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:07:49,447 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:49,447 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:07:49,447 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:49,447 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:07:49,791 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:50,111 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:07:52,389 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:52,389 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:52,389 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:07:52,389 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:07:52,389 INFO SenderThread:1740211 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:07:52,392 INFO SenderThread:1740211 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:07:52,393 INFO SenderThread:1740211 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:07:52,404 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:07:52,404 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:52,404 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:07:52,404 DEBUG SenderThread:1740211 [sender.py:send():379] send: artifact +2024-12-31 21:07:52,753 INFO wandb-upload_1:1740211 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaag6j_j3 +2024-12-31 21:07:52,793 INFO Thread-12 :1740211 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:52,941 INFO wandb-upload_0:1740211 [upload_job.py:push():88] Uploaded file /tmp/tmp3rvw6g5e/wandb-job.json +2024-12-31 21:07:52,942 INFO wandb-upload_2:1740211 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8_hforgk +2024-12-31 21:07:53,112 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExNTMxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDExMjAxNQ==', 'versionIndex': 389}}} +2024-12-31 21:07:53,477 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:07:53,477 INFO SenderThread:1740211 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:07:53,793 INFO SenderThread:1740211 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml config.yaml +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json wandb-summary.json +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt requirements.txt +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log output.log +2024-12-31 21:07:53,794 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/diff.patch diff.patch +2024-12-31 21:07:53,796 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 media/videos/eval/4_eval_4_c43d65e2ea74e852f815.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 media/videos/eval/1_eval_1_53a45cc58516bfa82806.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 media/videos/eval/2_eval_2_fe1c0520eab448503096.mp4 +2024-12-31 21:07:53,798 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/3_eval_3_a0e47918912079386856.mp4 media/videos/eval/3_eval_3_a0e47918912079386856.mp4 +2024-12-31 21:07:53,801 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 media/videos/eval/0_eval_0_e739aa3a1064d7a115c6.mp4 +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json media/table/eval/total_success_rate_ci_6_9152843e0aab0648a7f1.table.json +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2024-12-31 21:07:53,803 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:07:53,803 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:53,803 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:53,803 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:07:53,804 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:53,804 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:07:53,804 INFO SenderThread:1740211 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:07:53,970 INFO wandb-upload_0:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/wandb-summary.json +2024-12-31 21:07:54,008 INFO wandb-upload_3:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/requirements.txt +2024-12-31 21:07:54,029 INFO wandb-upload_1:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/config.yaml +2024-12-31 21:07:54,031 INFO wandb-upload_2:1740211 [upload_job.py:push():130] Uploaded file /tmp/tmpj0365mt3wandb/3jni8frz-events.out.tfevents.1735697004.leonmkim-ROG-Strix-G15CS-G15CS.1740046.0 +2024-12-31 21:07:54,072 INFO wandb-upload_4:1740211 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/files/output.log +2024-12-31 21:07:54,113 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,113 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,272 INFO Thread-11 (_thread_body):1740211 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:07:54,272 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,272 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:07:54,273 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,273 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:07:54,273 INFO SenderThread:1740211 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:07:54,274 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:07:54,274 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,274 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:07:54,274 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,275 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:07:54,275 INFO SenderThread:1740211 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:07:54,347 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,347 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:07:54,347 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:07:54,347 INFO SenderThread:1740211 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:07:54,348 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:07:54,348 INFO HandlerThread:1740211 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send():379] send: final +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send():379] send: footer +2024-12-31 21:07:54,348 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: defer +2024-12-31 21:07:54,348 INFO SenderThread:1740211 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:07:54,348 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,349 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:07:54,349 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:07:54,349 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:07:54,351 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:07:54,351 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:07:54,401 DEBUG SenderThread:1740211 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:07:54,401 INFO MainThread:1740211 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:07:54,402 INFO MainThread:1740211 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:07:54,402 INFO MainThread:1740211 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:07:54,402 DEBUG HandlerThread:1740211 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:07:54,402 INFO HandlerThread:1740211 [handler.py:finish():882] shutting down handler +2024-12-31 21:07:55,352 INFO WriterThread:1740211 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [sender.py:finish():1615] shutting down sender +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:07:55,401 INFO SenderThread:1740211 [file_pusher.py:join():175] waiting for file pusher diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug.log b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d085c16e63e39bc828b3777514788f522d4338fe --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Configure stats pid to 1740046 +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/settings +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:03:23,427 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug.log +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/logs/debug-internal.log +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x'} +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():619] starting backend +2024-12-31 21:03:23,428 INFO MainThread:1740046 [wandb_init.py:init():623] setting up manager +2024-12-31 21:03:23,434 INFO MainThread:1740046 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:03:23,437 INFO MainThread:1740046 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:03:23,449 INFO MainThread:1740046 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:03:23,457 INFO MainThread:1740046 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:03:23,710 INFO MainThread:1740046 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:03:23,750 INFO MainThread:1740046 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:03:23,750 INFO MainThread:1740046 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:03:24,089 INFO MainThread:1740046 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:03:24,090 INFO MainThread:1740046 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:03:24,091 INFO MainThread:1740046 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nkdxms1x/tb, True +2024-12-31 21:03:29,760 INFO MainThread:1740046 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:07:55,403 WARNING MsgRouterThr:1740046 [router.py:message_loop():77] message_loop has been closed diff --git a/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c7fa8b0f7ae9f9d73ee8385dbe77ee8371952fe8 --- /dev/null +++ b/nkdxms1x/wandb/run-20241231_210323-nkdxms1x/run-nkdxms1x.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b93118725175056061c65a000c433999bf1f0914b65d11c18759a8684e9a26ed +size 255335 diff --git a/snapshot_4723.pt b/snapshot_4723.pt new file mode 100644 index 0000000000000000000000000000000000000000..579d7659b971c838bbab252b6c202582e3a24807 --- /dev/null +++ b/snapshot_4723.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5a904b3d230fa81f7d2b3d4ddb8d500bfae4e358683e3451bf9ea51a53a458e +size 910102360 diff --git a/wj867hap/.hydra/config.yaml b/wj867hap/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/wj867hap/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/wj867hap/.hydra/hydra.yaml b/wj867hap/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20cd9a856665e4f6f8ff97f8dce637fa30779f37 --- /dev/null +++ b/wj867hap/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/wj867hap/.hydra/overrides.yaml b/wj867hap/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/wj867hap/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/wj867hap/episode_rosbags/aligned_depth_to_color_K.npy b/wj867hap/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/wj867hap/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/wj867hap/episode_rosbags/cam_tf_world.npy b/wj867hap/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/wj867hap/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/wj867hap/episode_rosbags/color_K.npy b/wj867hap/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/wj867hap/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/wj867hap/episode_rosbags/depth_K.npy b/wj867hap/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/wj867hap/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/wj867hap/eval_robot.log b/wj867hap/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8d669b1da1e0716b8f521ad122a1b95c09c135e2 --- /dev/null +++ b/wj867hap/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:41:12,474][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:41:12,477][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/wj867hap/eval_video/0_eval.mp4 b/wj867hap/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae48f120acdc2f7643850669091d35204da1abe0 --- /dev/null +++ b/wj867hap/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d6f4da7bee0d1f957d994d5c8e510820f8b38b0b75470373fa974de6ed0baa +size 1152129 diff --git a/wj867hap/eval_video/1_eval.mp4 b/wj867hap/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e812aa0c6994fa9b9f5cb5a0f19f827ae864ead4 --- /dev/null +++ b/wj867hap/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b96c803a906c327b9d1e3334d7fd4516e1f03fa6780bcf56b2f556ea2053ccd9 +size 1158425 diff --git a/wj867hap/eval_video/2_eval.mp4 b/wj867hap/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d38e48c01eb9d7c06442367e83103ac170146e2 --- /dev/null +++ b/wj867hap/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612933313ae530de76995b3b4b305fea3566a9842889a326a5f66b7e77fd8b26 +size 1157510 diff --git a/wj867hap/eval_video/3_eval.mp4 b/wj867hap/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5935a8152f8b383a6d13242710e7b582122ac766 --- /dev/null +++ b/wj867hap/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10a7919af6282d6370962b1cb8bdde1001c6e4cf33ca9e7971ddcdd37f8815c +size 1156043 diff --git a/wj867hap/eval_video/4_eval.mp4 b/wj867hap/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a02798efcaefc79b11c72248a431e92842cfe84d --- /dev/null +++ b/wj867hap/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39687803b7dbf5e9f63de33900b7e59e37eb17996af8acb3cc8a900c758240bd +size 1218314 diff --git a/wj867hap/tb/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 b/wj867hap/tb/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7db91d3c8e7e9e4be0088e3764e9f02a1f3c9a4 --- /dev/null +++ b/wj867hap/tb/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23c64d31b090a967b0faa8930dcc3619b80057ac1913f6a879b61db0adb76486 +size 6539 diff --git a/wj867hap/wandb/debug-internal.log b/wj867hap/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3146230301a0cb1cefcb513f8f5d639f954274e6 --- /dev/null +++ b/wj867hap/wandb/debug-internal.log @@ -0,0 +1,769 @@ +2025-01-01 11:41:18,447 INFO StreamThr :1796618 [internal.py:wandb_internal():85] W&B internal server running at pid: 1796618, started at: 2025-01-01 11:41:18.447275 +2025-01-01 11:41:18,448 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:41:18,460 INFO WriterThread:1796618 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb +2025-01-01 11:41:18,461 DEBUG SenderThread:1796618 [sender.py:send():379] send: header +2025-01-01 11:41:18,470 DEBUG SenderThread:1796618 [sender.py:send():379] send: run +2025-01-01 11:41:18,809 INFO SenderThread:1796618 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files +2025-01-01 11:41:18,809 INFO SenderThread:1796618 [sender.py:_start_run_threads():1188] run started: wj867hap with start time 1735749678.447464 +2025-01-01 11:41:18,825 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:41:18,825 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:41:18,909 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:41:18,914 DEBUG HandlerThread:1796618 [system_info.py:__init__():26] System info init +2025-01-01 11:41:18,914 DEBUG HandlerThread:1796618 [system_info.py:__init__():41] System info init done +2025-01-01 11:41:18,914 INFO HandlerThread:1796618 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:41:18,914 INFO SystemMonitor:1796618 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:41:18,914 INFO HandlerThread:1796618 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:41:18,914 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:41:18,915 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:41:18,916 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:41:18,917 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:41:18,917 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:41:18,927 DEBUG HandlerThread:1796618 [system_info.py:probe():152] Probing system +2025-01-01 11:41:18,928 DEBUG HandlerThread:1796618 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:probe():200] Probing system done +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:41:18.927448', 'startedAt': '2025-01-01T16:41:18.434925', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.51325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4500.026, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.029, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 717.4390983581543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:41:18,933 INFO HandlerThread:1796618 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:41:18,933 INFO HandlerThread:1796618 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:_save_code():44] Saving code +2025-01-01 11:41:18,935 DEBUG HandlerThread:1796618 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:41:18,935 DEBUG HandlerThread:1796618 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:41:18,996 DEBUG HandlerThread:1796618 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:41:18,997 INFO HandlerThread:1796618 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:41:19,000 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:41:19,000 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:41:19,000 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:41:19,001 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:41:19,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:41:19,230 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:41:19,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:19,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:19,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:19,234 INFO HandlerThread:1796618 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/tb" + save: true +} +control { + relay_id: "baac21b36e9a41009f91349764ac9330" +} +_info { + stream_id: "wj867hap" +} + +2025-01-01 11:41:19,234 DEBUG HandlerThread:1796618 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:41:19,268 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/lmvnjdek-code/FISH/eval_robot.py +2025-01-01 11:41:19,269 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/u4eo03fr-diff.patch +2025-01-01 11:41:19,269 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/yln2tjqv-wandb-metadata.json +2025-01-01 11:41:19,364 DEBUG SenderThread:1796618 [sender.py:send():379] send: telemetry +2025-01-01 11:41:19,364 DEBUG SenderThread:1796618 [sender.py:send():379] send: tbrecord +2025-01-01 11:41:19,365 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:41:19,365 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 with policy live +2025-01-01 11:41:19,540 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/0rmxedzn-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:41:19,810 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH +2025-01-01 11:41:20,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:21,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:21,810 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:22,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:23,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:24,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:24,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:24,469 DEBUG SenderThread:1796618 [sender.py:send():379] send: config +2025-01-01 11:41:25,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:25,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:26,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:27,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:27,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:28,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:29,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:29,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:29,812 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:30,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:31,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:31,812 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:32,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:33,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:33,813 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:34,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:34,230 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:34,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:34,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:35,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:36,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:37,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:38,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:39,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:40,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:40,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:41,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:42,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:43,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:44,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:45,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:45,427 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:45,816 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:46,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:47,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:47,817 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:48,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:49,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:49,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:49,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:49,818 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:50,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:51,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:51,330 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:51,818 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:41:52,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:53,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:53,819 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:54,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:55,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:56,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:56,570 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:57,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:57,820 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:58,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:59,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:00,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:01,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:02,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:02,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:03,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:04,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:04,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:04,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:05,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:06,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:07,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:07,613 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:07,822 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:08,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:09,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:10,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:11,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:12,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:13,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:13,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:14,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:15,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:16,089 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:42:16,089 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 with policy now +2025-01-01 11:42:16,090 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:42:16,090 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:42:16,091 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:42:16,092 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:16,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:16,315 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/eat472xm-media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos +2025-01-01 11:42:17,082 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/6avs9boq-media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:17,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:17,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:18,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:18,917 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:42:18,918 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:42:18,918 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:19,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:19,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:19,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:20,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:21,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:21,826 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:22,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:23,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:23,827 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:23,949 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:24,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,828 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:26,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:27,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:28,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:28,829 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:42:29,056 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/6ulxl1fq-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:42:29,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:29,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:29,829 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:30,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:31,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:32,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:33,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:33,830 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:34,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:34,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:34,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:34,382 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:35,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:36,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:37,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:38,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:39,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:40,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:40,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:41,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:42,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:43,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:44,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:45,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:45,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:46,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:47,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:48,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:48,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:42:49,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:49,232 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:49,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:50,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:50,333 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:51,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:52,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:53,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:54,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:55,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:56,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:56,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:57,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:58,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:59,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:59,681 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:42:59,681 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 with policy now +2025-01-01 11:42:59,683 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:42:59,684 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:42:59,684 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:42:59,684 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:59,838 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:42:59,891 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/4gltlj4o-media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:43:00,151 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/rwt3inze-media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:43:00,249 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:01,250 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:01,707 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:01,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:02,251 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:03,250 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:04,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:04,239 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:04,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:05,251 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:05,840 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:06,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:07,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:07,573 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:07,841 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:08,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:08,688 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:08,688 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:08,688 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:08,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:08,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:09,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:09,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:10,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:11,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:11,843 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:12,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:13,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:13,422 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:13,844 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:14,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:15,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:16,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:17,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:18,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:18,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:43:18,919 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:19,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:19,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:19,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:20,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:21,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:22,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:23,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:24,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:24,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:25,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:26,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:27,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:28,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:29,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:29,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:30,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:31,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:32,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:33,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:34,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:34,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:34,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:34,319 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:35,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:36,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:37,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:38,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:39,257 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:39,940 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:40,263 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:40,345 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:43:40,346 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_612933313ae530de7699.mp4 with policy now +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:40,348 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:40,592 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/eqqza9sr-media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:43:41,111 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/4rz3u25z-media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:41,267 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:41,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:42,272 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:43,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:44,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:45,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:45,421 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:45,854 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:46,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:47,270 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:48,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:48,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:43:49,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:49,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:49,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:49,855 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:50,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,367 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:52,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:53,272 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:53,502 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:53,502 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:53,503 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:53,856 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:53,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:53,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:54,090 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/aycz2jez-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:54,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:56,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:56,948 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:57,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:57,858 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:58,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:59,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:59,858 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:00,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:01,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:02,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:02,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:03,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:04,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:04,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:04,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:05,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:05,861 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:06,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:07,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:07,386 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:08,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:09,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:10,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:11,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:12,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:13,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:13,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:14,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:15,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:16,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:17,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:18,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:18,920 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:44:18,921 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:19,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:19,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:19,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:20,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:21,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:22,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:23,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:23,939 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:24,283 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:25,285 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:25,476 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:44:25,476 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 with policy now +2025-01-01 11:44:25,479 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:44:25,480 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:44:25,480 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:25,480 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:25,686 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/0owi5c2e-media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:25,868 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:44:26,109 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/bnzt3dgq-media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:26,280 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:27,285 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:27,868 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:28,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:29,247 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:29,288 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:30,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:31,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:31,869 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:32,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,287 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,870 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:34,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:34,242 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:34,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:34,308 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:35,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:36,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:37,288 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:38,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:39,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:40,249 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:40,299 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:40,563 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:44:40,563 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:40,564 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:40,872 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:44:40,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:44:41,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:41,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:42,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:43,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:43,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:44,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:45,265 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:45,299 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:45,874 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:46,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:47,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:47,875 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:48,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:48,920 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:44:49,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:49,240 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:49,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,397 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:51,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:52,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:53,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:54,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:55,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:56,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:56,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:57,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:58,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:59,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:00,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:01,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:01,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:02,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:03,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:04,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:04,241 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:04,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:05,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,351 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:07,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:08,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:09,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:10,296 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:11,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:12,247 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:12,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:13,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:13,632 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:13,632 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 with policy now +2025-01-01 11:45:13,644 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:13,644 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:13,644 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:13,645 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:13,849 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/hh16onvh-media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:45:14,144 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/vzev4dmf-media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:14,307 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:15,304 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:15,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:16,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:17,260 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:17,312 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:18,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:18,921 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:45:19,248 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:19,249 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:19,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:19,886 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:20,309 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,886 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:22,306 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:22,581 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:23,310 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:24,327 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:25,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:26,325 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:27,328 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:27,490 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:45:27,490 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:45:27,635 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:27,888 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table +2025-01-01 11:45:27,933 INFO wandb-upload_0:1796618 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjsim9obi +2025-01-01 11:45:28,036 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:45:28,044 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:28,068 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/pz4kf5d6-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:28,158 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/sod26b89-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:28,302 INFO SenderThread:1796618 [sender.py:send_request_log_artifact():1525] logged artifact run-wj867hap-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5OTU1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDAwMw==', 'latestArtifact': None}} +2025-01-01 11:45:28,302 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:28,302 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:45:28,303 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:28,303 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:28,303 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,304 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send():379] send: telemetry +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,305 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,305 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,305 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,305 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,307 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,307 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,308 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,308 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,309 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,309 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,309 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,309 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,310 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,310 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,311 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,311 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:45:28,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:28,536 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/9mxcqlz0-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:28,670 INFO wandb-upload_1:1796618 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpld3wallo +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval +2025-01-01 11:45:29,184 INFO SenderThread:1796618 [sender.py:send_request_log_artifact():1525] logged artifact run-wj867hap-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5OTU2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDAwNA==', 'latestArtifact': None}} +2025-01-01 11:45:29,184 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:29,185 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json with policy now +2025-01-01 11:45:29,186 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:29,186 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:29,187 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:29,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:29,365 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/utrr2obo-media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:29,595 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/gqls6xyf-media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:29,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:29,892 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:30,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:31,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:32,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:33,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:34,241 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:34,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,331 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:35,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:36,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:37,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:38,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:39,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:40,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:40,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:40,896 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:45:41,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:42,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:43,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:44,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:45,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:46,113 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:46,161 DEBUG SenderThread:1796618 [sender.py:send():379] send: exit +2025-01-01 11:45:46,161 INFO SenderThread:1796618 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_exit():588] handling runtime: 267 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_exit():594] send defer +2025-01-01 11:45:46,162 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,162 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:45:46,162 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:45:46,163 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:45:46,163 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,163 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:45:46,163 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:45:46,163 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:45:46,164 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:45:46,164 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:45:46,217 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,217 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:45:46,218 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:45:46,218 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,218 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:45:46,898 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:47,899 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:45:52,630 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,630 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:45:52,630 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:52,631 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,632 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:45:52,633 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:52,633 DEBUG SenderThread:1796618 [sender.py:send():379] send: summary +2025-01-01 11:45:52,636 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,637 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,637 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:45:52,901 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:53,162 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:55,379 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:45:55,379 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:55,380 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,380 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:45:55,380 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,380 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:45:55,380 INFO SenderThread:1796618 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:45:55,382 INFO SenderThread:1796618 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:45:55,384 INFO SenderThread:1796618 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:45:55,394 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:45:55,394 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,394 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:45:55,394 DEBUG SenderThread:1796618 [sender.py:send():379] send: artifact +2025-01-01 11:45:55,586 INFO SenderThread:1796618 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:45:55,587 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,587 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:45:55,587 INFO SenderThread:1796618 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml config.yaml +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json wandb-summary.json +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt requirements.txt +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log output.log +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch diff.patch +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:45:55,907 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:45:55,907 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:45:55,908 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,909 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:45:55,911 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,911 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:45:55,911 INFO SenderThread:1796618 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:45:56,077 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:56,078 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt +2025-01-01 11:45:56,094 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:45:56,164 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,164 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,192 INFO wandb-upload_3:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/zpp257rz-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:56,200 INFO wandb-upload_4:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:56,400 INFO Thread-11 (_thread_body):1796618 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:45:56,400 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,401 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:45:56,401 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,401 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:45:56,401 INFO SenderThread:1796618 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:45:56,402 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,402 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:45:56,402 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:45:56,587 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,587 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:45:56,587 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:45:56,587 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,587 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send():379] send: final +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send():379] send: footer +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,588 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:45:56,588 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,589 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:45:56,589 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:45:56,589 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,590 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:45:56,591 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:45:56,591 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:56,640 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,640 INFO MainThread:1796618 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:45:56,641 INFO MainThread:1796618 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:45:56,641 INFO MainThread:1796618 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:45:56,641 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:45:56,641 INFO HandlerThread:1796618 [handler.py:finish():882] shutting down handler +2025-01-01 11:45:57,591 INFO WriterThread:1796618 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [sender.py:finish():1615] shutting down sender +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [file_pusher.py:join():175] waiting for file pusher diff --git a/wj867hap/wandb/debug.log b/wj867hap/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c961b339006b23084f7a8a08cdfe2519d5518057 --- /dev/null +++ b/wj867hap/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Configure stats pid to 1796442 +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/settings +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug.log +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug-internal.log +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap'} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():619] starting backend +2025-01-01 11:41:18,442 INFO MainThread:1796442 [wandb_init.py:init():623] setting up manager +2025-01-01 11:41:18,446 INFO MainThread:1796442 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:41:18,447 INFO MainThread:1796442 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:41:18,460 INFO MainThread:1796442 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:41:18,467 INFO MainThread:1796442 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:41:18,824 INFO MainThread:1796442 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:41:18,863 INFO MainThread:1796442 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:41:18,863 INFO MainThread:1796442 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:41:19,232 INFO MainThread:1796442 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:41:19,233 INFO MainThread:1796442 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:41:19,234 INFO MainThread:1796442 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/tb, True +2025-01-01 11:41:24,468 INFO MainThread:1796442 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 11:45:57,642 WARNING MsgRouterThr:1796442 [router.py:message_loop():77] message_loop has been closed diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py b/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml b/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6085ff7c34267bf7bc33e2d3efed46550e9b0995 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735749678 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch b/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..eff8de3c864ce1e6f5e6f3000cbf990769adf78e --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch @@ -0,0 +1,105 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..b9e3dfe 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history ++# wandb_run_id: '1002_0' ++ ++# all books, 6/20 demos per book ++# RGBD+mask+act history + wandb_run_id: '1007_0' + ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..53ca8b7c1df7a633773294e16e559ba06828abcb --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[28, 55, 0.509090909090909, 0.3525364697274449, 0.6631653606565301, 0.381764404499336, 0.6456638723320505, 0.4017411917644561, 0.6156984083123077, 0.4178573936388269, 0.601098842898653]]} \ No newline at end of file diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae48f120acdc2f7643850669091d35204da1abe0 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d6f4da7bee0d1f957d994d5c8e510820f8b38b0b75470373fa974de6ed0baa +size 1152129 diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e812aa0c6994fa9b9f5cb5a0f19f827ae864ead4 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b96c803a906c327b9d1e3334d7fd4516e1f03fa6780bcf56b2f556ea2053ccd9 +size 1158425 diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d38e48c01eb9d7c06442367e83103ac170146e2 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612933313ae530de76995b3b4b305fea3566a9842889a326a5f66b7e77fd8b26 +size 1157510 diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5935a8152f8b383a6d13242710e7b582122ac766 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10a7919af6282d6370962b1cb8bdde1001c6e4cf33ca9e7971ddcdd37f8815c +size 1156043 diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a02798efcaefc79b11c72248a431e92842cfe84d --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39687803b7dbf5e9f63de33900b7e59e37eb17996af8acb3cc8a900c758240bd +size 1218314 diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log b/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fc2dfd5aa94f4cbc8a9a250ed351828e05a1fbb7 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735749684.469125]: resetting environment +[INFO] [1735749684.472346]: cleared current plan +[INFO] [1735749684.473343]: moving to home +[INFO] [1735749685.475366]: reached home +[INFO] [1735749685.476712]: reset action history +[INFO] [1735749688.186954]: environment reset +Starting episode 0 +[INFO] [1735749688.188990]: resetting environment +[INFO] [1735749688.193493]: cleared current plan +[INFO] [1735749688.194487]: moving to home +[INFO] [1735749689.196871]: reached home +[INFO] [1735749689.198272]: reset action history +[INFO] [1735749691.906690]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735749704.420588]: resetting environment +[INFO] [1735749704.423927]: cleared current plan +[INFO] [1735749704.424810]: moving to home +[INFO] [1735749705.426485]: reached home +[INFO] [1735749705.427820]: reset action history +[INFO] [1735749708.137766]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/episode_rosbags/episode_0_2025-01-01-11-41-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1796442.log +started roslaunch server http://158.130.50.37:35833/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1796719] +process[print_text-2]: started with pid [1796743] +process[rosbag_record-3]: started with pid [1796744] +started bagging! +For topic gripper_width: timestamp difference is -111935718 for nearest: 1735749711578617470 - target: 1735749711690553188 at idx 55 +For topic gripper_width: timestamp difference is -111521914 for nearest: 1735749711778622196 - target: 1735749711890144110 at idx 56 +For topic gripper_width: timestamp difference is -102103694 for nearest: 1735749716378628032 - target: 1735749716480731726 at idx 119 +For topic EE_pose: timestamp difference is 113617258 for nearest: 1735749725179029303 - target: 1735749725065412045 at idx 0 +[INFO] [1735749735.150340]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735749735.890211]: Stored episode 1. +[INFO] [1735749735.890425]: Saving video... +[INFO] [1735749736.079299]: Video saved! +Starting episode 1 +[INFO] [1735749736.090734]: resetting environment +[INFO] [1735749736.095013]: cleared current plan +[INFO] [1735749736.095407]: moving to home +[INFO] [1735749740.197268]: reached home +[INFO] [1735749740.198313]: reset action history +[INFO] [1735749742.906446]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749747.863846]: resetting environment +[INFO] [1735749747.868649]: cleared current plan +[INFO] [1735749747.868852]: moving to home +[INFO] [1735749748.870444]: reached home +[INFO] [1735749748.870854]: reset action history +[INFO] [1735749751.577507]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/episode_rosbags/episode_1_2025-01-01-11-42-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1796442.log +started roslaunch server http://158.130.50.37:46339/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1796854] +process[print_text-5]: started with pid [1796855] +process[rosbag_record-6]: started with pid [1796879] +started bagging! +[INFO] [1735749778.563700]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735749779.299649]: Stored episode 2. +[INFO] [1735749779.300254]: Saving video... +[INFO] [1735749779.655177]: Video saved! +Starting episode 2 +[INFO] [1735749779.685567]: resetting environment +[INFO] [1735749779.697450]: cleared current plan +[INFO] [1735749779.705035]: moving to home +[INFO] [1735749783.810916]: reached home +[INFO] [1735749783.813547]: reset action history +[INFO] [1735749786.529574]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749788.400379]: resetting environment +[INFO] [1735749788.408563]: cleared current plan +[INFO] [1735749788.408906]: moving to home +[INFO] [1735749789.410728]: reached home +[INFO] [1735749789.411686]: reset action history +[INFO] [1735749792.171538]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/episode_rosbags/episode_2_2025-01-01-11-43-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1796442.log +started roslaunch server http://158.130.50.37:42379/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1797001] +process[print_text-8]: started with pid [1797002] +process[rosbag_record-9]: started with pid [1797003] +started bagging! +[INFO] [1735749819.126856]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735749819.937673]: Stored episode 3. +[INFO] [1735749819.944256]: Saving video... +[INFO] [1735749820.296290]: Video saved! +Starting episode 3 +[INFO] [1735749820.342687]: resetting environment +[INFO] [1735749820.355304]: cleared current plan +[INFO] [1735749820.363869]: moving to home +[INFO] [1735749824.375975]: reached home +[INFO] [1735749824.380228]: reset action history +[INFO] [1735749827.150893]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749833.046523]: resetting environment +[INFO] [1735749833.059056]: cleared current plan +[INFO] [1735749833.061451]: moving to home +[INFO] [1735749834.064199]: reached home +[INFO] [1735749834.067377]: reset action history +[INFO] [1735749836.947789]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/episode_rosbags/episode_3_2025-01-01-11-43-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1796442.log +started roslaunch server http://158.130.50.37:36137/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1797136] +process[print_text-11]: started with pid [1797160] +process[rosbag_record-12]: started with pid [1797161] +started bagging! +For topic gripper_width: timestamp difference is -102934542 for nearest: 1735749843178597030 - target: 1735749843281531572 at idx 94 +[INFO] [1735749863.937377]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735749864.766064]: Stored episode 4. +[INFO] [1735749864.770708]: Saving video... +[WARN] [1735749865.289417]: Plan exhausted +[WARN] [1735749865.325669]: Plan exhausted +[WARN] [1735749865.369997]: Plan exhausted +[WARN] [1735749865.408985]: Plan exhausted +[WARN] [1735749865.449252]: Plan exhausted +[WARN] [1735749865.486092]: Plan exhausted +[INFO] [1735749865.445328]: Video saved! +Starting episode 4 +[INFO] [1735749865.480935]: resetting environment +[INFO] [1735749865.492219]: cleared current plan +[INFO] [1735749865.494038]: moving to home +[INFO] [1735749869.504171]: reached home +[INFO] [1735749869.512987]: reset action history +[INFO] [1735749872.295830]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749880.409807]: resetting environment +[INFO] [1735749880.422884]: cleared current plan +[INFO] [1735749880.426085]: moving to home +[INFO] [1735749881.435003]: reached home +[INFO] [1735749881.440191]: reset action history +[INFO] [1735749884.219844]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/episode_rosbags/episode_4_2025-01-01-11-44-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1796442.log +started roslaunch server http://158.130.50.37:34883/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1797297] +process[print_text-14]: started with pid [1797298] +process[rosbag_record-15]: started with pid [1797299] +started bagging! +[INFO] [1735749912.570507]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit[rosbag_record-15] killing on exit +[INFO] [1735749913.345293]: Stored episode 5. +[INFO] [1735749913.350379]: Saving video... +[INFO] [1735749913.587192]: Video saved! +[INFO] [1735749913.645088]: resetting environment +[INFO] [1735749913.664723]: cleared current plan +[INFO] [1735749913.667755]: moving to home +[INFO] [1735749917.671208]: reached home +[INFO] [1735749917.683252]: reset action history +[INFO] [1735749920.535539]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt b/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json b/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..96c3e8a1a2a895695a5f44332b277fb25d97863b --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:41:18.927448", + "startedAt": "2025-01-01T16:41:18.434925", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.51325, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.026, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.029, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.994, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 717.4390983581543 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json b/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fdcd021a29acb854ff5fa184f0c355ac680cd371 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "88d6f4da7bee0d1f957d994d5c8e510820f8b38b0b75470373fa974de6ed0baa", "size": 1152129, "path": "media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4"}, "global_step": 664, "_timestamp": 1735749927.937453, "_runtime": 249.4899890422821, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b96c803a906c327b9d1e3334d7fd4516e1f03fa6780bcf56b2f556ea2053ccd9", "size": 1158425, "path": "media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4"}, "eval/num_success": 5.0, "episode": 55.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "612933313ae530de76995b3b4b305fea3566a9842889a326a5f66b7e77fd8b26", "size": 1157510, "path": "media/videos/eval/2_eval_2_612933313ae530de7699.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f10a7919af6282d6370962b1cb8bdde1001c6e4cf33ca9e7971ddcdd37f8815c", "size": 1156043, "path": "media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "39687803b7dbf5e9f63de33900b7e59e37eb17996af8acb3cc8a900c758240bd", "size": 1218314, "path": "media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://pps89sedz1cnvj8nxrcpjjfsogc3p5w78ejhztw19bilhkvd6ax5jkv9tpwqjcwev4p8zrkfyctgw8n07csv2s0grw0uhluc44jeki74uo3i2qiv62vmclw1t4l9s6sg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://onorh2z2xniyiwhuismhpgdjwdoszsq52io8gmiolp6csjhc3dd787lr2mv3uisf4j7uvjqq58ztrqua0qe6ruvx30yznceky5w64gjs8mion10wjat2rsr0tdgipwqb:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 28, "total_num_episodes": 55, "total_success_rate": 0.509090909090909, "total_success_rate_lb_0.01": 0.3525364697274449, "total_success_rate_ub_0.01": 0.6631653606565301, "total_success_rate_lb_0.025": 0.381764404499336, "total_success_rate_ub_0.025": 0.6456638723320505, "total_success_rate_lb_0.05": 0.4017411917644561, "total_success_rate_ub_0.05": 0.6156984083123077, "total_success_rate_lb_0.1": 0.4178573936388269, "total_success_rate_ub_0.1": 0.601098842898653, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "18f444fb6de2e2c5af7b922f95b728ca0948ff60faf34f553c39b1fd8bbe31cb", "size": 517, "artifact_path": "wandb-client-artifact://j87vixcoi9uijk63k12ilqbbtt8jlkwbdao6y8vwllwdmkgfjmucsrrasq77aqfu41660cqhj4qel27dokosjqn4uv9bs7o7k0hbw63u4r984vgoidfzl1mf31qxgcet/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ci895yoiilkfjghlf9y2kdkl5sxu5pymhu3yn1lc9wiaqrjelmmmsvoqrl16bpjkrc9hq8punfh76pstmhdzmabhkxbcpjrefzteayxgswhnd5oc4wixy136f7kp0pgd:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 267}, "eval/total_success_rate": 0.5090909004211426, "eval/total_success_rate_lb_0.01": 0.35253646969795227, "eval/total_success_rate_ub_0.01": 0.6631653308868408, "eval/total_success_rate_lb_0.025": 0.38176441192626953, "eval/total_success_rate_ub_0.025": 0.645663857460022, "eval/total_success_rate_lb_0.05": 0.4017412066459656, "eval/total_success_rate_ub_0.05": 0.6156983971595764, "eval/total_success_rate_lb_0.1": 0.41785740852355957, "eval/total_success_rate_ub_0.1": 0.6010988354682922} \ No newline at end of file diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug-internal.log b/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3146230301a0cb1cefcb513f8f5d639f954274e6 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug-internal.log @@ -0,0 +1,769 @@ +2025-01-01 11:41:18,447 INFO StreamThr :1796618 [internal.py:wandb_internal():85] W&B internal server running at pid: 1796618, started at: 2025-01-01 11:41:18.447275 +2025-01-01 11:41:18,448 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:41:18,460 INFO WriterThread:1796618 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb +2025-01-01 11:41:18,461 DEBUG SenderThread:1796618 [sender.py:send():379] send: header +2025-01-01 11:41:18,470 DEBUG SenderThread:1796618 [sender.py:send():379] send: run +2025-01-01 11:41:18,809 INFO SenderThread:1796618 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files +2025-01-01 11:41:18,809 INFO SenderThread:1796618 [sender.py:_start_run_threads():1188] run started: wj867hap with start time 1735749678.447464 +2025-01-01 11:41:18,825 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:41:18,825 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:41:18,909 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:41:18,914 DEBUG HandlerThread:1796618 [system_info.py:__init__():26] System info init +2025-01-01 11:41:18,914 DEBUG HandlerThread:1796618 [system_info.py:__init__():41] System info init done +2025-01-01 11:41:18,914 INFO HandlerThread:1796618 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:41:18,914 INFO SystemMonitor:1796618 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:41:18,914 INFO HandlerThread:1796618 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:41:18,914 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:41:18,915 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:41:18,916 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:41:18,917 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:41:18,917 INFO SystemMonitor:1796618 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:41:18,927 DEBUG HandlerThread:1796618 [system_info.py:probe():152] Probing system +2025-01-01 11:41:18,928 DEBUG HandlerThread:1796618 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:probe():200] Probing system done +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:41:18.927448', 'startedAt': '2025-01-01T16:41:18.434925', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.51325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4500.026, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.029, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 717.4390983581543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:41:18,933 INFO HandlerThread:1796618 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:41:18,933 INFO HandlerThread:1796618 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:41:18,933 DEBUG HandlerThread:1796618 [system_info.py:_save_code():44] Saving code +2025-01-01 11:41:18,935 DEBUG HandlerThread:1796618 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:41:18,935 DEBUG HandlerThread:1796618 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:41:18,996 DEBUG HandlerThread:1796618 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:41:18,997 INFO HandlerThread:1796618 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:41:19,000 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:41:19,000 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:41:19,000 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:41:19,001 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:41:19,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:41:19,230 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:41:19,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:19,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:19,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:19,234 INFO HandlerThread:1796618 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/tb" + save: true +} +control { + relay_id: "baac21b36e9a41009f91349764ac9330" +} +_info { + stream_id: "wj867hap" +} + +2025-01-01 11:41:19,234 DEBUG HandlerThread:1796618 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:41:19,268 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/lmvnjdek-code/FISH/eval_robot.py +2025-01-01 11:41:19,269 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/u4eo03fr-diff.patch +2025-01-01 11:41:19,269 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/yln2tjqv-wandb-metadata.json +2025-01-01 11:41:19,364 DEBUG SenderThread:1796618 [sender.py:send():379] send: telemetry +2025-01-01 11:41:19,364 DEBUG SenderThread:1796618 [sender.py:send():379] send: tbrecord +2025-01-01 11:41:19,365 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:41:19,365 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 with policy live +2025-01-01 11:41:19,540 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/0rmxedzn-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:41:19,810 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code +2025-01-01 11:41:19,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH +2025-01-01 11:41:20,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:21,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:21,810 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:22,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:23,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:24,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:24,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:24,469 DEBUG SenderThread:1796618 [sender.py:send():379] send: config +2025-01-01 11:41:25,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:25,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:26,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:27,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:27,811 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:28,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:29,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:29,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:29,812 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:30,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:31,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:31,812 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:32,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:33,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:33,813 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:34,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:34,230 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:34,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:34,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:35,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:36,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:37,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:38,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:39,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:40,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:40,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:41,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:42,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:43,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:44,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:45,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:45,427 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:45,816 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:46,234 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:47,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:47,817 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:48,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:49,230 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:41:49,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:41:49,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:49,818 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:50,235 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:51,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:51,330 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:51,818 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:41:52,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:53,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:53,819 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:54,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:55,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:56,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:56,570 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:41:57,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:57,820 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:41:58,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:41:59,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:00,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:01,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:02,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:02,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:03,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:04,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:04,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:04,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:05,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:06,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:07,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:07,613 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:07,822 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:08,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:09,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:10,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:11,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:12,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:13,232 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:13,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:14,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:15,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:16,089 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:42:16,089 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 with policy now +2025-01-01 11:42:16,090 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:42:16,090 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:42:16,091 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:42:16,092 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:16,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:16,315 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/eat472xm-media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media +2025-01-01 11:42:16,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos +2025-01-01 11:42:17,082 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/6avs9boq-media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:42:17,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:17,825 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:18,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:18,917 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:42:18,918 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:42:18,918 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:19,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:19,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:19,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:20,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:21,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:21,826 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:22,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:23,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:23,827 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:23,949 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:24,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:25,828 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:26,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:27,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:28,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:28,829 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:42:29,056 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/6ulxl1fq-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:42:29,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:29,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:29,829 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:30,242 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:31,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:32,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:33,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:33,830 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:34,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:34,231 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:34,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:34,382 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:35,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:36,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:37,243 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:38,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:39,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:40,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:40,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:41,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:42,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:43,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:44,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:45,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:45,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:46,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:47,244 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:48,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:48,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:42:49,231 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:42:49,232 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:42:49,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:50,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:50,333 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:51,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:52,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:53,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:54,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:55,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:56,233 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:42:56,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:57,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:58,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:59,246 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:42:59,681 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:42:59,681 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 with policy now +2025-01-01 11:42:59,683 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:42:59,684 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:42:59,684 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:42:59,684 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:42:59,838 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:42:59,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:42:59,891 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/4gltlj4o-media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:43:00,151 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/rwt3inze-media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:43:00,249 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:01,250 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:01,707 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:01,839 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:02,251 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:03,250 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:04,239 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:04,239 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:04,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:05,251 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:05,840 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:06,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:07,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:07,573 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:07,841 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:08,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:08,688 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:08,688 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:08,688 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:08,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:08,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:09,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:09,842 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:10,252 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:11,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:11,843 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:12,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:13,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:13,422 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:13,844 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:14,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:15,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:16,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:17,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:18,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:18,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:43:18,919 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:19,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:19,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:19,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:20,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:21,254 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:22,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:23,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:24,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:24,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:25,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:26,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:27,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:28,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:29,238 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:29,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:30,255 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:31,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:32,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:33,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:34,236 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:34,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:34,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:34,319 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:35,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:36,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:37,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:38,256 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:39,257 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:39,940 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:40,263 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:40,345 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:43:40,346 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_612933313ae530de7699.mp4 with policy now +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:40,346 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:40,348 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:40,592 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/eqqza9sr-media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:40,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:43:41,111 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/4rz3u25z-media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:43:41,267 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:41,852 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:42,272 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:43,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:44,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:45,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:45,421 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:45,854 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:46,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:47,270 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:48,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:48,919 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:43:49,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:43:49,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:43:49,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:49,855 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:50,271 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:51,367 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:52,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:53,272 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:53,502 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:43:53,502 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:43:53,503 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:43:53,856 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:53,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:53,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:43:54,090 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/aycz2jez-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:43:54,273 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:55,857 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:56,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:56,948 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:43:57,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:57,858 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:43:58,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:59,274 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:43:59,858 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:00,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:01,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:02,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:02,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:03,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:04,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:04,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:04,275 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:05,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:05,861 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:06,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:07,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:07,386 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:08,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:09,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:10,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:11,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:12,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:13,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:13,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:14,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:15,276 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:16,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:17,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:18,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:18,920 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:44:18,921 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:19,237 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:19,237 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:19,281 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:20,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:21,277 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:22,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:23,278 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:23,939 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:24,283 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:25,285 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:25,476 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:44:25,476 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 with policy now +2025-01-01 11:44:25,479 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:44:25,480 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:44:25,480 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:25,480 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:25,686 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/0owi5c2e-media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:44:25,867 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:25,868 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:44:26,109 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/bnzt3dgq-media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:44:26,280 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:27,285 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:27,868 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:28,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:29,247 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:29,288 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:30,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:31,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:31,869 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:32,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,287 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:33,870 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:34,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:34,242 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:34,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:34,308 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:35,289 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:36,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:37,288 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:38,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:39,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:40,249 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:40,299 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:40,563 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:44:40,563 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:44:40,564 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:44:40,872 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:44:40,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:44:41,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:41,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:42,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:43,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:43,873 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:44,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:45,265 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:45,299 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:45,874 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:46,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:47,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:47,875 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:44:48,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:48,920 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:44:49,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:44:49,240 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:44:49,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,292 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:50,397 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:51,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:52,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:53,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:54,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:55,293 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:56,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:44:56,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:57,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:58,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:44:59,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:00,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:01,245 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:01,294 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:02,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:03,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:04,240 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:04,241 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:04,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:05,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:06,351 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:07,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:08,295 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:09,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:10,296 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:11,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:12,247 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:12,297 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:13,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:13,632 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:13,632 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 with policy now +2025-01-01 11:45:13,644 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:13,644 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:13,644 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:13,645 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:13,849 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/hh16onvh-media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:13,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval +2025-01-01 11:45:14,144 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/vzev4dmf-media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:14,307 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:15,304 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:15,884 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:16,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:17,260 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:17,312 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:18,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:18,921 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:45:19,248 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:19,249 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:19,305 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:19,886 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:20,309 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:21,886 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:22,306 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:22,581 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:23,310 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:24,327 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:25,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:26,325 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:27,328 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:27,490 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:45:27,490 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:45:27,635 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:27,888 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval +2025-01-01 11:45:27,889 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table +2025-01-01 11:45:27,933 INFO wandb-upload_0:1796618 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjsim9obi +2025-01-01 11:45:28,036 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:45:28,044 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:45:28,068 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/pz4kf5d6-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:28,158 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/sod26b89-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:28,302 INFO SenderThread:1796618 [sender.py:send_request_log_artifact():1525] logged artifact run-wj867hap-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5OTU1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDAwMw==', 'latestArtifact': None}} +2025-01-01 11:45:28,302 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:28,302 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:45:28,303 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:28,303 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,303 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:28,303 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,304 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send():379] send: telemetry +2025-01-01 11:45:28,304 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,305 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,305 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,305 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,305 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,307 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,307 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,308 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,308 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,308 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,309 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,309 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,309 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,309 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,310 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,310 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,310 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:28,311 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:28,311 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:45:28,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:28,536 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/9mxcqlz0-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:28,670 INFO wandb-upload_1:1796618 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpld3wallo +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:28,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval +2025-01-01 11:45:29,184 INFO SenderThread:1796618 [sender.py:send_request_log_artifact():1525] logged artifact run-wj867hap-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5OTU2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDAwNA==', 'latestArtifact': None}} +2025-01-01 11:45:29,184 DEBUG SenderThread:1796618 [sender.py:send():379] send: files +2025-01-01 11:45:29,185 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json with policy now +2025-01-01 11:45:29,186 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:29,186 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:29,187 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:29,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:29,365 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/utrr2obo-media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:29,595 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/gqls6xyf-media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:29,891 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:29,892 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:30,320 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:31,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:32,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:33,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,241 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:45:34,241 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:45:34,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:34,331 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:35,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:36,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:37,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:38,321 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:39,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:40,253 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:40,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:40,896 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:45:41,322 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:42,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:43,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:44,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:45,323 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:46,113 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:46,161 DEBUG SenderThread:1796618 [sender.py:send():379] send: exit +2025-01-01 11:45:46,161 INFO SenderThread:1796618 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_exit():588] handling runtime: 267 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_exit():594] send defer +2025-01-01 11:45:46,162 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,162 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:45:46,162 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:45:46,162 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:45:46,163 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:45:46,163 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,163 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:45:46,163 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:45:46,163 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:45:46,163 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:45:46,164 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:45:46,164 DEBUG SystemMonitor:1796618 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:45:46,216 INFO HandlerThread:1796618 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send():379] send: stats +2025-01-01 11:45:46,217 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,217 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:45:46,217 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:46,217 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:45:46,218 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:45:46,218 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:46,218 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:45:46,898 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:47,899 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:52,629 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:45:52,630 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,630 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:45:52,630 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send():379] send: history +2025-01-01 11:45:52,631 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:45:52,631 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,632 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:45:52,633 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:52,633 DEBUG SenderThread:1796618 [sender.py:send():379] send: summary +2025-01-01 11:45:52,636 INFO SenderThread:1796618 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,637 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:45:52,637 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:52,637 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:45:52,637 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:52,637 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:45:52,901 INFO Thread-12 :1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:53,162 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:55,379 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:45:55,379 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:55,380 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,380 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:45:55,380 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,380 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:45:55,380 INFO SenderThread:1796618 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:45:55,382 INFO SenderThread:1796618 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:45:55,384 INFO SenderThread:1796618 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:45:55,394 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:45:55,394 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,394 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:45:55,394 DEBUG SenderThread:1796618 [sender.py:send():379] send: artifact +2025-01-01 11:45:55,586 INFO SenderThread:1796618 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:45:55,587 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,587 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:45:55,587 INFO SenderThread:1796618 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml config.yaml +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json wandb-summary.json +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt requirements.txt +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:45:55,902 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log output.log +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/diff.patch diff.patch +2025-01-01 11:45:55,903 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:45:55,907 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 media/videos/eval/3_eval_3_f10a7919af6282d63709.mp4 +2025-01-01 11:45:55,907 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 media/videos/eval/0_eval_0_88d6f4da7bee0d1f957d.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/2_eval_2_612933313ae530de7699.mp4 media/videos/eval/2_eval_2_612933313ae530de7699.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 media/videos/eval/1_eval_1_b96c803a906c327b9d1e.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 media/videos/eval/4_eval_4_39687803b7dbf5e9f63d.mp4 +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json media/table/eval/total_success_rate_ci_6_18f444fb6de2e2c5af7b.table.json +2025-01-01 11:45:55,908 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:45:55,908 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:55,909 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:45:55,911 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:55,911 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:45:55,911 INFO SenderThread:1796618 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:45:56,077 INFO wandb-upload_0:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/wandb-summary.json +2025-01-01 11:45:56,078 INFO wandb-upload_1:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/requirements.txt +2025-01-01 11:45:56,094 INFO wandb-upload_2:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/config.yaml +2025-01-01 11:45:56,164 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,164 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,192 INFO wandb-upload_3:1796618 [upload_job.py:push():130] Uploaded file /tmp/tmpxs9mtz2wwandb/zpp257rz-events.out.tfevents.1735749679.leonmkim-ROG-Strix-G15CS-G15CS.1796442.0 +2025-01-01 11:45:56,200 INFO wandb-upload_4:1796618 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/files/output.log +2025-01-01 11:45:56,400 INFO Thread-11 (_thread_body):1796618 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:45:56,400 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,401 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:45:56,401 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,401 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:45:56,401 INFO SenderThread:1796618 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:45:56,402 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,402 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:45:56,402 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:45:56,402 INFO SenderThread:1796618 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:45:56,587 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,587 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:45:56,587 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:45:56,587 INFO SenderThread:1796618 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:45:56,587 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:45:56,587 INFO HandlerThread:1796618 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send():379] send: final +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send():379] send: footer +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: defer +2025-01-01 11:45:56,588 INFO SenderThread:1796618 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:45:56,588 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,588 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,589 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:45:56,589 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:45:56,589 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:45:56,590 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:45:56,591 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:45:56,591 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:45:56,640 DEBUG SenderThread:1796618 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:45:56,640 INFO MainThread:1796618 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:45:56,641 INFO MainThread:1796618 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:45:56,641 INFO MainThread:1796618 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:45:56,641 DEBUG HandlerThread:1796618 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:45:56,641 INFO HandlerThread:1796618 [handler.py:finish():882] shutting down handler +2025-01-01 11:45:57,591 INFO WriterThread:1796618 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [sender.py:finish():1615] shutting down sender +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:45:57,640 INFO SenderThread:1796618 [file_pusher.py:join():175] waiting for file pusher diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug.log b/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c961b339006b23084f7a8a08cdfe2519d5518057 --- /dev/null +++ b/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Configure stats pid to 1796442 +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/settings +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug.log +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/wandb/run-20250101_114118-wj867hap/logs/debug-internal.log +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap'} +2025-01-01 11:41:18,441 INFO MainThread:1796442 [wandb_init.py:init():619] starting backend +2025-01-01 11:41:18,442 INFO MainThread:1796442 [wandb_init.py:init():623] setting up manager +2025-01-01 11:41:18,446 INFO MainThread:1796442 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:41:18,447 INFO MainThread:1796442 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:41:18,460 INFO MainThread:1796442 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:41:18,467 INFO MainThread:1796442 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:41:18,824 INFO MainThread:1796442 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:41:18,863 INFO MainThread:1796442 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:41:18,863 INFO MainThread:1796442 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:41:19,231 INFO MainThread:1796442 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:41:19,232 INFO MainThread:1796442 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:41:19,233 INFO MainThread:1796442 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:41:19,234 INFO MainThread:1796442 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/wj867hap/tb, True +2025-01-01 11:41:24,468 INFO MainThread:1796442 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 11:45:57,642 WARNING MsgRouterThr:1796442 [router.py:message_loop():77] message_loop has been closed diff --git a/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb b/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb new file mode 100644 index 0000000000000000000000000000000000000000..86cb98c58119200f065f7d630597c0cebf9fcee7 Binary files /dev/null and b/wj867hap/wandb/run-20250101_114118-wj867hap/run-wj867hap.wandb differ