diff --git a/.gitattributes b/.gitattributes index e248dcf6c12a196f734e7abb7048b70b822eb65d..792916063520290211c16af3711fca37a0eab8ec 100644 --- a/.gitattributes +++ b/.gitattributes @@ -152,3 +152,34 @@ iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag filter=lfs diff=lfs m 8qih81be/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text 8qih81be/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag filter=lfs diff=lfs merge=lfs -text +8qih81be/episode_rosbags/episode_3_2025-01-01-11-26-14.bag filter=lfs diff=lfs merge=lfs -text +iz44h5u3/episode_rosbags/episode_0_2025-01-01-11-52-32.bag filter=lfs diff=lfs merge=lfs -text +8qih81be/episode_rosbags/episode_1_2025-01-01-11-24-16.bag filter=lfs diff=lfs merge=lfs -text +k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/episode_rosbags/episode_2_2025-01-01-11-25-18.bag filter=lfs diff=lfs merge=lfs -text +k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/episode_rosbags/episode_4_2025-01-01-11-27-11.bag filter=lfs diff=lfs merge=lfs -text +k7tzwekb/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/episode_rosbags/episode_1_2025-01-01-00-08-20.bag filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/episode_rosbags/episode_4_2025-01-01-00-12-40.bag filter=lfs diff=lfs merge=lfs -text +z0y7vp39/episode_rosbags/episode_3_2024-12-31-21-31-12.bag filter=lfs diff=lfs merge=lfs -text +z0y7vp39/episode_rosbags/episode_4_2024-12-31-21-33-47.bag filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yr3ixpt5/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k7tzwekb/episode_rosbags/episode_2_2024-12-31-23-34-24.bag filter=lfs diff=lfs merge=lfs -text +6rp1ws1k/episode_rosbags/episode_1_2024-12-31-22-29-18.bag filter=lfs diff=lfs merge=lfs -text diff --git a/6rp1ws1k/episode_rosbags/episode_1_2024-12-31-22-29-18.bag b/6rp1ws1k/episode_rosbags/episode_1_2024-12-31-22-29-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..4f035be79e95f72b3d4886354faad0acfe318c9b --- /dev/null +++ b/6rp1ws1k/episode_rosbags/episode_1_2024-12-31-22-29-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb3f970e896ea71c7ea8fb59a81bcc6e3955fc9011e2befa0c6f7535e71eebdd +size 2605586328 diff --git a/7j9cp4xk/episode_rosbags/episode_1_2025-01-01-00-08-20.bag b/7j9cp4xk/episode_rosbags/episode_1_2025-01-01-00-08-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..d121f286a42071e033ad7babb88c75c3a644a94f --- /dev/null +++ b/7j9cp4xk/episode_rosbags/episode_1_2025-01-01-00-08-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390eb7524fcabe856a0eb1eae45a18b92efd1648baaa52ff95304aa8318c0b85 +size 2608752244 diff --git a/7j9cp4xk/episode_rosbags/episode_4_2025-01-01-00-12-40.bag b/7j9cp4xk/episode_rosbags/episode_4_2025-01-01-00-12-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..ee9491db73e1ab6d94054717c1c8548b127b7ed5 --- /dev/null +++ b/7j9cp4xk/episode_rosbags/episode_4_2025-01-01-00-12-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df86d9711c6b346b2fcfdf1584e646c4611ff19bd20afe2525415d2a75abe2b5 +size 2603419953 diff --git a/8qih81be/episode_rosbags/episode_1_2025-01-01-11-24-16.bag b/8qih81be/episode_rosbags/episode_1_2025-01-01-11-24-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..9c881e7093a48b3677281595c8400650646d9a3b --- /dev/null +++ b/8qih81be/episode_rosbags/episode_1_2025-01-01-11-24-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cb5cf4daaceda2e22ea2817f31f3420dff17fa15dd88f2c35af04845ad0eb8c +size 1417465693 diff --git a/8qih81be/episode_rosbags/episode_2_2025-01-01-11-25-18.bag b/8qih81be/episode_rosbags/episode_2_2025-01-01-11-25-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..852fb5a5fde664a2d35ffaecdb9c12e00da52741 --- /dev/null +++ b/8qih81be/episode_rosbags/episode_2_2025-01-01-11-25-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4c754860c23817b13c0fad9d37426d4bb64fc70049c4c3294bc9715f9e59bc9 +size 1628602772 diff --git a/8qih81be/episode_rosbags/episode_3_2025-01-01-11-26-14.bag b/8qih81be/episode_rosbags/episode_3_2025-01-01-11-26-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..01cef786ffe866d66c866c6cefc88bb90cf514f2 --- /dev/null +++ b/8qih81be/episode_rosbags/episode_3_2025-01-01-11-26-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38214aef11268f23e5308eb1bb1e43265e99ffd23f002999dfab2f0c4a2924fc +size 1353174457 diff --git a/8qih81be/episode_rosbags/episode_4_2025-01-01-11-27-11.bag b/8qih81be/episode_rosbags/episode_4_2025-01-01-11-27-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..5b1af22b9b78f9099a2d629354e0a44a1cb21e7b --- /dev/null +++ b/8qih81be/episode_rosbags/episode_4_2025-01-01-11-27-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf541b282197ce52097ecd9bc14e02db6bf4a667985dc00d6482050c494ee44 +size 1695478565 diff --git a/ivv3ndjr/episode_rosbags/cam_tf_world.npy b/ivv3ndjr/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ivv3ndjr/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ivv3ndjr/episode_rosbags/depth_K.npy b/ivv3ndjr/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ivv3ndjr/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ivv3ndjr/eval_robot.log b/ivv3ndjr/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..d7d1690abc72d5d0641309a32157d3e1178b33b3 --- /dev/null +++ b/ivv3ndjr/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:08:47,672][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:08:47,675][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/iz44h5u3/episode_rosbags/episode_0_2025-01-01-11-52-32.bag b/iz44h5u3/episode_rosbags/episode_0_2025-01-01-11-52-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..47fb55debc6c9617fd3d9d3f9454b75b171bdf3a --- /dev/null +++ b/iz44h5u3/episode_rosbags/episode_0_2025-01-01-11-52-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390e05a21252a80ff4c6e39da83780fba7dba2a0a5bca099e3ceca7b6b80011d +size 2616539389 diff --git a/k7tzwekb/.hydra/config.yaml b/k7tzwekb/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/k7tzwekb/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/k7tzwekb/.hydra/hydra.yaml b/k7tzwekb/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6522268b42c44d8059e443309358183feeb172b7 --- /dev/null +++ b/k7tzwekb/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/k7tzwekb/.hydra/overrides.yaml b/k7tzwekb/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/k7tzwekb/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/k7tzwekb/episode_rosbags/aligned_depth_to_color_K.npy b/k7tzwekb/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k7tzwekb/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k7tzwekb/episode_rosbags/color_K.npy b/k7tzwekb/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k7tzwekb/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k7tzwekb/episode_rosbags/episode_2_2024-12-31-23-34-24.bag b/k7tzwekb/episode_rosbags/episode_2_2024-12-31-23-34-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..14482119ec7bd35f2723f323b95653137fe6c61f --- /dev/null +++ b/k7tzwekb/episode_rosbags/episode_2_2024-12-31-23-34-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03c8db9df3858c3d39eb701492d515fd7e2491c070558d7af9e5b0c0ea06350 +size 2601847565 diff --git a/k7tzwekb/eval_video/0_eval.mp4 b/k7tzwekb/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79ce77f6aee14f349652163b03679c2b30947d31 --- /dev/null +++ b/k7tzwekb/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:595f5f50e069a5ef6f724ce9d9d5ac9943f0fe878037ae02d7851a8ceee49cbb +size 1455806 diff --git a/k7tzwekb/eval_video/1_eval.mp4 b/k7tzwekb/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5e4a6e0507e5534759d4d32db822b149e4fa277 --- /dev/null +++ b/k7tzwekb/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0681f94ab8a1121a68ec757371dc803fd132f9a2250d3b36f5e5365762eb9e +size 1491773 diff --git a/k7tzwekb/eval_video/2_eval.mp4 b/k7tzwekb/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48de9d8fc6599fdbe5bcf071cd22fb0407a94cc3 --- /dev/null +++ b/k7tzwekb/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f6ca98f28c7680329f2630bc15d430932da760438314f06003ac8b317b2693 +size 1509827 diff --git a/k7tzwekb/eval_video/3_eval.mp4 b/k7tzwekb/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..badb3a08fad68d27e6f3cddc65ebee5a021391a3 --- /dev/null +++ b/k7tzwekb/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a59939926caab3bf8cabd65841b37a48821355d5327cf8a328bbd6bc329f00e0 +size 1490802 diff --git a/k7tzwekb/eval_video/4_eval.mp4 b/k7tzwekb/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6664c436ab01d4d13215b3a6dd04e890470bbdd --- /dev/null +++ b/k7tzwekb/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece347de7301c56db2359fb44a224add3a73a3615c92c68c7e8e2f6c4927f172 +size 1560314 diff --git a/k7tzwekb/tb/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 b/k7tzwekb/tb/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 new file mode 100644 index 0000000000000000000000000000000000000000..0883aa97ccc41b52a719b7f1cce8efbbea74ece7 --- /dev/null +++ b/k7tzwekb/tb/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f24b0bc2d7c28bb69a2c2d5d20246e2f6d2c87fe66c1b68d45c1246d746efd +size 6539 diff --git a/k7tzwekb/wandb/debug-internal.log b/k7tzwekb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0121ae69299bd6f73654031a3ecbfb2d748442c7 --- /dev/null +++ b/k7tzwekb/wandb/debug-internal.log @@ -0,0 +1,797 @@ +2024-12-31 23:32:23,261 INFO StreamThr :1765812 [internal.py:wandb_internal():85] W&B internal server running at pid: 1765812, started at: 2024-12-31 23:32:23.260807 +2024-12-31 23:32:23,263 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:32:23,273 INFO WriterThread:1765812 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb +2024-12-31 23:32:23,274 DEBUG SenderThread:1765812 [sender.py:send():379] send: header +2024-12-31 23:32:23,286 DEBUG SenderThread:1765812 [sender.py:send():379] send: run +2024-12-31 23:32:23,577 INFO SenderThread:1765812 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files +2024-12-31 23:32:23,577 INFO SenderThread:1765812 [sender.py:_start_run_threads():1188] run started: k7tzwekb with start time 1735705943.262408 +2024-12-31 23:32:23,592 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:32:23,593 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:32:23,635 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:32:23,640 DEBUG HandlerThread:1765812 [system_info.py:__init__():26] System info init +2024-12-31 23:32:23,640 DEBUG HandlerThread:1765812 [system_info.py:__init__():41] System info init done +2024-12-31 23:32:23,640 INFO HandlerThread:1765812 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:32:23,640 INFO SystemMonitor:1765812 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:32:23,640 INFO HandlerThread:1765812 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:32:23,641 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:32:23,641 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:32:23,642 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:32:23,642 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:32:23,643 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:32:23,653 DEBUG HandlerThread:1765812 [system_info.py:probe():152] Probing system +2024-12-31 23:32:23,655 DEBUG HandlerThread:1765812 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:probe():200] Probing system done +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:32:23.654015', 'startedAt': '2025-01-01T04:32:23.252341', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.9963749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.006, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 585.1749725341797}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:32:23,660 INFO HandlerThread:1765812 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:32:23,660 INFO HandlerThread:1765812 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:_save_code():44] Saving code +2024-12-31 23:32:23,661 DEBUG HandlerThread:1765812 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:32:23,662 DEBUG HandlerThread:1765812 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:32:23,723 DEBUG HandlerThread:1765812 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:32:23,724 INFO HandlerThread:1765812 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:32:23,727 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:32:23,727 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:32:23,727 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:32:23,728 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:32:23,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:32:23,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:32:23,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:23,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:23,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:23,968 INFO HandlerThread:1765812 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/tb" + save: true +} +control { + relay_id: "730dfe5eef704cfb8b1e23cb2ed50623" +} +_info { + stream_id: "k7tzwekb" +} + +2024-12-31 23:32:23,968 DEBUG HandlerThread:1765812 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:32:23,992 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/ryq7cqux-wandb-metadata.json +2024-12-31 23:32:23,996 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/fhupt9xd-diff.patch +2024-12-31 23:32:24,017 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/4hiq4fyd-code/FISH/eval_robot.py +2024-12-31 23:32:24,049 DEBUG SenderThread:1765812 [sender.py:send():379] send: telemetry +2024-12-31 23:32:24,050 DEBUG SenderThread:1765812 [sender.py:send():379] send: tbrecord +2024-12-31 23:32:24,050 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:32:24,050 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 with policy live +2024-12-31 23:32:24,240 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/l1hvbsv9-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:32:24,578 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH +2024-12-31 23:32:24,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:25,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:26,578 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:26,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:27,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:28,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:28,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:29,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:30,831 DEBUG SenderThread:1765812 [sender.py:send():379] send: config +2024-12-31 23:32:30,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:31,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:32,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:32,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:33,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:33,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:34,580 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:34,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:35,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:36,581 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:36,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:37,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:38,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:38,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:38,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:39,023 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:39,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:40,582 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:40,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:41,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:42,582 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:42,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:43,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:44,182 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:44,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:45,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:46,583 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:46,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:47,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:48,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:49,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:49,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:50,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:51,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:52,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:53,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:53,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:54,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:54,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:55,028 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:55,586 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:32:55,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:56,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:57,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:58,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:59,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:00,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:00,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:01,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:02,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:03,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:04,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:05,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:06,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:06,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:07,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:08,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:08,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:09,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:09,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:10,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:11,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:12,112 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:12,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:13,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:14,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:15,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:16,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:17,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:17,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:18,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:19,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:20,593 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:20,671 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:33:20,671 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 with policy now +2024-12-31 23:33:20,671 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:33:20,672 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:33:20,673 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:33:20,673 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:33:20,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:21,020 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/ko0udlk1-media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media +2024-12-31 23:33:21,946 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/19027wpk-media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:22,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:22,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:23,643 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:33:23,644 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:33:23,644 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:23,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:23,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:24,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:24,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:25,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:26,596 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:26,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:27,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:28,597 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:28,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:28,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:29,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:30,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:31,598 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:33:31,859 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/fbp5oznv-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:33:31,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:32,598 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:32,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:33,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:34,599 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:34,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:34,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:35,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:36,599 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:36,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:37,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:38,600 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:38,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:38,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:39,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:39,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:40,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:40,979 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:41,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:42,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:43,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:44,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:45,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:45,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:46,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:47,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:48,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:49,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:50,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:50,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:51,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:52,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:53,644 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:33:53,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:53,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:54,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:54,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:55,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:56,074 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:56,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:57,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:58,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:59,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:00,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:01,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:01,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:02,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:03,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:04,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:05,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:06,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:06,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:07,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:08,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:08,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:09,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:09,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:10,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:11,985 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:12,296 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:12,540 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:34:12,570 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 with policy now +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:34:12,582 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:34:12,616 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:34:12,616 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:12,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:12,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:34:12,883 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/5avac1s8-media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:12,985 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:13,241 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/id2rq8kr-media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:13,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:14,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:14,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:15,989 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:16,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:17,521 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:17,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:18,618 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:18,989 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:19,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:20,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:34:20,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:20,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:21,378 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:34:21,378 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:34:21,378 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:34:21,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:34:21,994 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:22,609 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:22,621 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:22,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:23,645 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:34:23,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:23,969 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:24,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:24,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:25,995 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:26,622 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:26,995 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:27,652 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:27,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:28,623 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:28,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:29,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:30,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:31,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:32,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:32,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:33,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:34,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:35,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:36,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:37,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:38,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:38,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:39,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:39,083 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:39,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:40,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:41,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:42,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:43,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:44,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:44,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:45,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:47,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:48,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:48,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:49,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:50,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:51,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:52,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:53,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:53,645 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:34:53,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:53,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:54,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:55,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:55,082 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:56,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:57,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:58,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:59,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:00,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:00,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:01,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:02,005 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:02,209 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:35:02,209 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 with policy now +2024-12-31 23:35:02,213 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:35:02,214 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:02,214 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:02,221 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:02,546 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/4w0lfl6k-media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:02,634 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:35:02,947 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/f5l8nbpp-media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:03,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:04,012 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:04,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:05,014 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:05,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:06,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:07,020 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:08,023 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:08,636 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:08,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:08,977 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:09,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:10,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:10,637 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:11,022 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:11,112 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:12,020 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:12,222 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:12,222 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:12,223 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:12,638 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:12,638 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:35:12,918 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/wnuar5hj-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:35:13,022 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:14,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:14,639 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:15,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:16,031 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:16,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:17,028 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:18,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:18,640 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:19,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:20,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:20,641 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:21,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:21,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:22,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:23,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:23,646 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:35:23,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:23,968 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:24,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:25,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:26,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:27,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:27,124 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:28,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:29,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:30,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:31,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:32,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:32,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:33,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:34,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:35,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:36,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:37,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:37,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:38,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:38,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:38,968 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:39,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:40,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:41,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:42,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:43,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:43,104 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:44,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:45,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:46,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:47,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:48,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:48,325 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:48,650 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:49,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:50,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:51,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:52,038 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:53,049 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:53,365 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:53,607 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:35:53,607 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 with policy now +2024-12-31 23:35:53,612 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:35:53,612 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:53,612 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:53,634 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:53,646 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:35:53,904 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/b87wtpcs-media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:53,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:53,975 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:54,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:54,204 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/73j44g99-media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:54,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:55,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:56,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:57,046 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:58,044 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:58,609 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:58,654 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:59,043 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:00,044 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:01,050 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:02,045 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:02,655 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:03,046 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:03,886 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:04,043 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:04,289 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:04,289 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:04,289 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:04,656 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:04,656 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:05,049 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:06,056 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:06,657 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:07,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:08,056 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:08,657 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:08,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:08,974 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:09,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:09,129 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:10,113 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:10,658 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:11,061 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:12,061 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:12,659 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:13,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:14,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:14,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:15,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:16,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:17,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:18,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:19,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:19,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:20,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:21,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:22,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:23,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:23,647 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:36:23,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:23,972 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:24,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:25,048 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:25,089 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:26,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:26,664 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:27,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:28,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:29,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:30,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:30,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:31,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:32,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:33,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:34,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:35,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:35,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:36,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:37,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:38,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:38,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:38,972 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:39,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:40,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:41,066 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:41,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:42,067 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:42,669 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:43,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:44,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:45,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:46,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:46,938 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:46,938 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 with policy now +2024-12-31 23:36:46,941 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:46,941 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:46,941 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:46,957 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:46,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:47,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:47,230 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/uvyj7fkc-media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:36:48,004 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/crbdudzu-media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:48,073 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:48,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:49,073 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:50,075 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:51,088 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:52,083 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:52,673 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:52,933 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:53,082 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:53,647 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:36:53,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:53,990 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:54,084 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:54,673 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:55,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:56,100 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:56,357 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:56,357 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:56,358 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:56,674 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:56,674 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:56,942 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/uxtoxhk4-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:57,084 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:36:57,084 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:36:57,087 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:57,263 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:57,538 INFO wandb-upload_0:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_qqzkyke +2024-12-31 23:36:57,653 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:36:57,665 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table +2024-12-31 23:36:57,955 INFO SenderThread:1765812 [sender.py:send_request_log_artifact():1525] logged artifact run-k7tzwekb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwMzc5NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODU2NA==', 'latestArtifact': None}} +2024-12-31 23:36:57,955 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:57,955 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 23:36:57,956 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:57,956 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:57,956 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,957 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,957 DEBUG SenderThread:1765812 [sender.py:send():379] send: telemetry +2024-12-31 23:36:57,957 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,961 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,961 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,961 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,961 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,962 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,963 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,963 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,964 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,964 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,965 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,965 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,966 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,966 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,967 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,967 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:36:58,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:58,141 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/lqf7it18-media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:58,163 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/kb400lzc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:58,330 INFO wandb-upload_0:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_7f0h_k4 +2024-12-31 23:36:58,616 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/u9ug01rg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:58,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:58,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:send_request_log_artifact():1525] logged artifact run-k7tzwekb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwMzgwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODU2OA==', 'latestArtifact': None}} +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json with policy now +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:59,043 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/g9xp0dmb-media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:59,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:59,676 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:00,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:00,676 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:01,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:02,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:02,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:03,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:04,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:05,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:06,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:07,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:08,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:08,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:37:08,978 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:37:09,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:09,138 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:09,558 DEBUG SenderThread:1765812 [sender.py:send():379] send: exit +2024-12-31 23:37:09,558 INFO SenderThread:1765812 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:37:09,558 INFO SenderThread:1765812 [sender.py:send_exit():588] handling runtime: 285 +2024-12-31 23:37:09,559 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:09,559 INFO SenderThread:1765812 [sender.py:send_exit():594] send defer +2024-12-31 23:37:09,559 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,559 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:37:09,559 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:37:09,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:37:09,560 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:37:09,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:37:09,560 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:37:09,560 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:37:09,561 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:37:09,561 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:37:09,613 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,613 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:37:09,613 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:37:09,614 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:37:09,614 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,614 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:37:09,614 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,614 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:37:09,614 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:37:09,614 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,614 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:37:09,679 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:10,680 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:15,682 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,355 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:37:16,355 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:37:16,356 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,356 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:37:16,356 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:16,357 DEBUG SenderThread:1765812 [sender.py:send():379] send: summary +2024-12-31 23:37:16,360 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:16,360 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,361 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:37:16,361 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,361 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:37:16,361 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:37:16,559 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:16,683 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:18,349 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:37:18,349 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:18,349 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:18,349 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:37:18,350 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:18,350 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:37:18,350 INFO SenderThread:1765812 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:37:18,352 INFO SenderThread:1765812 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:37:18,354 INFO SenderThread:1765812 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:37:18,367 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:37:18,368 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:18,368 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:37:18,368 DEBUG SenderThread:1765812 [sender.py:send():379] send: artifact +2024-12-31 23:37:18,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:18,683 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:18,765 INFO wandb-upload_1:1765812 [upload_job.py:push():85] Skipped uploading /tmp/tmpqmi6n6n8/wandb-job.json +2024-12-31 23:37:18,766 INFO wandb-upload_0:1765812 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3oy2c3ah +2024-12-31 23:37:18,861 INFO wandb-upload_2:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf0rhrj42 +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNDA4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE5ODAyNA==', 'versionIndex': 399}}} +2024-12-31 23:37:19,593 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:37:19,684 INFO SenderThread:1765812 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files +2024-12-31 23:37:19,684 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml config.yaml +2024-12-31 23:37:19,685 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json wandb-summary.json +2024-12-31 23:37:19,686 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt requirements.txt +2024-12-31 23:37:19,686 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:37:19,687 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:37:19,687 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log output.log +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch diff.patch +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:37:19,692 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:19,692 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:19,694 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:37:19,694 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:19,694 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:37:19,694 INFO SenderThread:1765812 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:37:19,871 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt +2024-12-31 23:37:19,887 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:37:19,888 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:19,943 INFO wandb-upload_4:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:20,006 INFO wandb-upload_3:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/7vcbs7zi-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:37:20,206 INFO Thread-11 (_thread_body):1765812 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:37:20,207 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,207 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:37:20,207 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,207 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:37:20,207 INFO SenderThread:1765812 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:37:20,208 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:37:20,208 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,208 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:37:20,208 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,209 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:37:20,209 INFO SenderThread:1765812 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:37:20,266 INFO SenderThread:1765812 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:37:20,266 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:37:20,266 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,266 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:37:20,267 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,267 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send():379] send: final +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send():379] send: footer +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:20,268 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:20,269 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:37:20,269 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:37:20,270 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:37:20,271 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:20,325 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:20,325 INFO MainThread:1765812 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:37:20,326 INFO MainThread:1765812 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:37:20,326 INFO MainThread:1765812 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:37:20,326 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:37:20,326 INFO HandlerThread:1765812 [handler.py:finish():882] shutting down handler +2024-12-31 23:37:21,269 INFO WriterThread:1765812 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb +2024-12-31 23:37:21,325 INFO SenderThread:1765812 [sender.py:finish():1615] shutting down sender +2024-12-31 23:37:21,326 INFO SenderThread:1765812 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:37:21,326 INFO SenderThread:1765812 [file_pusher.py:join():175] waiting for file pusher diff --git a/k7tzwekb/wandb/debug.log b/k7tzwekb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fed0b9c1b65cf7edd5cd372061c48d23dcda7847 --- /dev/null +++ b/k7tzwekb/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:32:23,254 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Configure stats pid to 1765571 +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/settings +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug.log +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug-internal.log +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb'} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():619] starting backend +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():623] setting up manager +2024-12-31 23:32:23,260 INFO MainThread:1765571 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:32:23,262 INFO MainThread:1765571 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:32:23,273 INFO MainThread:1765571 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:32:23,284 INFO MainThread:1765571 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:32:23,592 INFO MainThread:1765571 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:32:23,631 INFO MainThread:1765571 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:32:23,631 INFO MainThread:1765571 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:32:23,967 INFO MainThread:1765571 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:32:23,968 INFO MainThread:1765571 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/tb, True +2024-12-31 23:32:30,830 INFO MainThread:1765571 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 23:37:21,327 WARNING MsgRouterThr:1765571 [router.py:message_loop():77] message_loop has been closed diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dd2a14cbd9b5af30790c5100e29eff326cee428 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735705943 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..609dac12f6049dd7c1c940762e170874fb625fea --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..3db3932 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 15.0 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json new file mode 100644 index 0000000000000000000000000000000000000000..65afadc60678f5b0acadbe46560fa3e29c036a6f --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 20, 0.75, 0.4809804260768783, 0.9234382807361589, 0.5409779846586287, 0.9123561232457377, 0.5756642520276376, 0.8764129354796986, 0.5938638672044902, 0.8696323334431482]]} \ No newline at end of file diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79ce77f6aee14f349652163b03679c2b30947d31 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:595f5f50e069a5ef6f724ce9d9d5ac9943f0fe878037ae02d7851a8ceee49cbb +size 1455806 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5e4a6e0507e5534759d4d32db822b149e4fa277 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0681f94ab8a1121a68ec757371dc803fd132f9a2250d3b36f5e5365762eb9e +size 1491773 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48de9d8fc6599fdbe5bcf071cd22fb0407a94cc3 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f6ca98f28c7680329f2630bc15d430932da760438314f06003ac8b317b2693 +size 1509827 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..badb3a08fad68d27e6f3cddc65ebee5a021391a3 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a59939926caab3bf8cabd65841b37a48821355d5327cf8a328bbd6bc329f00e0 +size 1490802 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6664c436ab01d4d13215b3a6dd04e890470bbdd --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece347de7301c56db2359fb44a224add3a73a3615c92c68c7e8e2f6c4927f172 +size 1560314 diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7173fb60ba5712e673060ba6222e2e79361f5af5 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log @@ -0,0 +1,262 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735705950.831382]: resetting environment +[INFO] [1735705950.839013]: cleared current plan +[INFO] [1735705950.839882]: moving to home +[INFO] [1735705951.842029]: reached home +[INFO] [1735705951.845142]: reset action history +[INFO] [1735705954.556221]: environment reset +Starting episode 0 +[INFO] [1735705954.560765]: resetting environment +[INFO] [1735705954.566751]: cleared current plan +[INFO] [1735705954.567811]: moving to home +[INFO] [1735705955.569264]: reached home +[INFO] [1735705955.570743]: reset action history +[INFO] [1735705958.331941]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735705960.462698]: resetting environment +[INFO] [1735705960.469224]: cleared current plan +[INFO] [1735705960.470198]: moving to home +[INFO] [1735705961.471905]: reached home +[INFO] [1735705961.473304]: reset action history +[INFO] [1735705964.181709]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/episode_rosbags/episode_0_2024-12-31-23-32-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1765571.log +started roslaunch server http://158.130.50.37:33581/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1765957] +process[print_text-2]: started with pid [1765958] +process[rosbag_record-3]: started with pid [1765959] +started bagging! +[INFO] [1735705999.559565]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735706000.372397]: Stored episode 1. +[INFO] [1735706000.372609]: Saving video... +[INFO] [1735706000.657370]: Video saved! +Starting episode 1 +[INFO] [1735706000.672468]: resetting environment +[INFO] [1735706000.711996]: cleared current plan +[INFO] [1735706000.712273]: moving to home +[INFO] [1735706004.615091]: reached home +[INFO] [1735706004.615568]: reset action history +[INFO] [1735706007.332015]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706011.450825]: resetting environment +[INFO] [1735706011.462281]: cleared current plan +[INFO] [1735706011.462487]: moving to home +[INFO] [1735706012.464018]: reached home +[INFO] [1735706012.464513]: reset action history +[INFO] [1735706015.174776]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/episode_rosbags/episode_1_2024-12-31-23-33-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1765571.log +started roslaunch server http://158.130.50.37:37769/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1766187] +process[print_text-5]: started with pid [1766188] +process[rosbag_record-6]: started with pid [1766189] +started bagging! +[INFO] [1735706051.440441]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735706052.294552]: Stored episode 2. +[INFO] [1735706052.300378]: Saving video... +[INFO] [1735706052.514007]: Video saved! +Starting episode 2 +[INFO] [1735706052.545470]: resetting environment +[INFO] [1735706052.571293]: cleared current plan +[INFO] [1735706052.571674]: moving to home +[INFO] [1735706056.475856]: reached home +[INFO] [1735706056.479314]: reset action history +[INFO] [1735706059.251439]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706060.550604]: resetting environment +[INFO] [1735706060.559799]: cleared current plan +[INFO] [1735706060.559980]: moving to home +[INFO] [1735706061.561826]: reached home +[INFO] [1735706061.564957]: reset action history +[INFO] [1735706064.356461]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/episode_rosbags/episode_2_2024-12-31-23-34-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1765571.log +started roslaunch server http://158.130.50.37:35451/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1766706] +process[print_text-8]: started with pid [1766707] +process[rosbag_record-9]: started with pid [1766708] +started bagging! +[INFO] [1735706101.022757]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735706101.930347]: Stored episode 3. +[INFO] [1735706101.940339]: Saving video... +[INFO] [1735706102.166872]: Video saved! +Starting episode 3 +[INFO] [1735706102.218014]: resetting environment +[INFO] [1735706102.240756]: cleared current plan +[INFO] [1735706102.241008]: moving to home +[INFO] [1735706106.147413]: reached home +[INFO] [1735706106.150060]: reset action history +[INFO] [1735706108.932520]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706112.117063]: resetting environment +[INFO] [1735706112.144935]: cleared current plan +[INFO] [1735706112.145164]: moving to home +[INFO] [1735706113.148957]: reached home +[INFO] [1735706113.151868]: reset action history +[INFO] [1735706115.934568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/episode_rosbags/episode_3_2024-12-31-23-35-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1765571.log +started roslaunch server http://158.130.50.37:38807/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1766913] +process[print_text-11]: started with pid [1766914] +process[rosbag_record-12]: started with pid [1766915] +started bagging! +For topic gripper_width: timestamp difference is -136586463 for nearest: 1735706146053342069 - target: 1735706146189928532 at idx 249 +[INFO] [1735706152.512994]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735706153.321022]: Stored episode 4. +[INFO] [1735706153.335090]: Saving video... +[INFO] [1735706153.543491]: Video saved! +Starting episode 4 +[INFO] [1735706153.613910]: resetting environment +[INFO] [1735706153.642086]: cleared current plan +[INFO] [1735706153.644018]: moving to home +[INFO] [1735706157.566747]: reached home +[INFO] [1735706157.574146]: reset action history +[INFO] [1735706160.375406]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706163.943213]: resetting environment +[INFO] [1735706163.976889]: cleared current plan +[INFO] [1735706163.977143]: moving to home +[INFO] [1735706164.979034]: reached home +[INFO] [1735706164.982379]: reset action history +[INFO] [1735706167.785541]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/episode_rosbags/episode_4_2024-12-31-23-36-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1765571.log +started roslaunch server http://158.130.50.37:35997/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1767136] +process[print_text-14]: started with pid [1767137] +process[rosbag_record-15]: started with pid [1767138] +started bagging! +For topic gripper_width: timestamp difference is -111154602 for nearest: 1735706184853281452 - target: 1735706184964436054 at idx 219 +For topic gripper_width: timestamp difference is 100879779 for nearest: 1735706199919952741 - target: 1735706199819072962 at idx 249 +[INFO] [1735706205.942806]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735706206.661900]: Stored episode 5. +[INFO] [1735706206.668655]: Saving video... +[INFO] [1735706206.903909]: Video saved! +[INFO] [1735706206.940427]: resetting environment +[INFO] [1735706206.973086]: cleared current plan +[INFO] [1735706206.973814]: moving to home +[INFO] [1735706210.881212]: reached home +[INFO] [1735706210.896896]: reset action history +[INFO] [1735706213.707150]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dccad0d6bddff440462162ef6737c9d4ae6cfd4a --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:32:23.654015", + "startedAt": "2025-01-01T04:32:23.252341", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.9963749999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.014, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.994, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.006, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 585.1749725341797 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d2a04721e847ae458751c070ad46c0c73a0b5f --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "595f5f50e069a5ef6f724ce9d9d5ac9943f0fe878037ae02d7851a8ceee49cbb", "size": 1455806, "path": "media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4"}, "global_step": 665, "_timestamp": 1735706217.5533822, "_runtime": 274.29097414016724, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "fa0681f94ab8a1121a68ec757371dc803fd132f9a2250d3b36f5e5365762eb9e", "size": 1491773, "path": "media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a8f6ca98f28c7680329f2630bc15d430932da760438314f06003ac8b317b2693", "size": 1509827, "path": "media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a59939926caab3bf8cabd65841b37a48821355d5327cf8a328bbd6bc329f00e0", "size": 1490802, "path": "media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ece347de7301c56db2359fb44a224add3a73a3615c92c68c7e8e2f6c4927f172", "size": 1560314, "path": "media/videos/eval/4_eval_4_ece347de7301c56db235.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://tgthtr3b4yczxsa8dqmuvy008lugdit620dedtr92z8hss2qsdwq8pvj30czopod6g0zniocdtgiv35vbxbow5y201vbo56a2yog07kkvlkgr4wf8q5q8q7a0v2x780d/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gy0ab9h9ri06l60v2ybiz4au0r7lxag6bui9o0bfwoi942snm70jqgli0ovzyv6bbyag4nt1tru76r2bhct835f0b4qdod6kqjz9wsfnk1zspenxr3sp1ci9peubz6zk:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 20, "total_success_rate": 0.75, "total_success_rate_lb_0.01": 0.4809804260768783, "total_success_rate_ub_0.01": 0.9234382807361589, "total_success_rate_lb_0.025": 0.5409779846586287, "total_success_rate_ub_0.025": 0.9123561232457377, "total_success_rate_lb_0.05": 0.5756642520276376, "total_success_rate_ub_0.05": 0.8764129354796986, "total_success_rate_lb_0.1": 0.5938638672044902, "total_success_rate_ub_0.1": 0.8696323334431482, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "a60f92e596343da90112641cd3fb1b2f8abe3876549c90a19a0a08bf7131710b", "size": 506, "artifact_path": "wandb-client-artifact://j4h8us8dtf4zlfmt9g7d5v5pgak34rjlhhm9imkhjr84ntdt1gpon24w5z7m4bdldi7uglsr2vtsb30okyasp3astnq9hhemji8b98m3y8lobfupbv750w00mgjnklvm/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r92cztcf2jeztd1bmo37jdcl5y8aof9cepl4q16d59cf81vfuej6ugx8u7xmqq8iigq46dq5wua87815slu52p3rvr1p47iirxm2v8y75mxesat3lz8779ozn7uy5y5d:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 285}, "eval/total_success_rate": 0.75, "eval/total_success_rate_lb_0.01": 0.48098042607307434, "eval/total_success_rate_ub_0.01": 0.9234382510185242, "eval/total_success_rate_lb_0.025": 0.540977954864502, "eval/total_success_rate_ub_0.025": 0.9123561382293701, "eval/total_success_rate_lb_0.05": 0.575664222240448, "eval/total_success_rate_ub_0.05": 0.8764129281044006, "eval/total_success_rate_lb_0.1": 0.593863844871521, "eval/total_success_rate_ub_0.1": 0.8696323037147522} \ No newline at end of file diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug-internal.log b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0121ae69299bd6f73654031a3ecbfb2d748442c7 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug-internal.log @@ -0,0 +1,797 @@ +2024-12-31 23:32:23,261 INFO StreamThr :1765812 [internal.py:wandb_internal():85] W&B internal server running at pid: 1765812, started at: 2024-12-31 23:32:23.260807 +2024-12-31 23:32:23,263 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:32:23,273 INFO WriterThread:1765812 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb +2024-12-31 23:32:23,274 DEBUG SenderThread:1765812 [sender.py:send():379] send: header +2024-12-31 23:32:23,286 DEBUG SenderThread:1765812 [sender.py:send():379] send: run +2024-12-31 23:32:23,577 INFO SenderThread:1765812 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files +2024-12-31 23:32:23,577 INFO SenderThread:1765812 [sender.py:_start_run_threads():1188] run started: k7tzwekb with start time 1735705943.262408 +2024-12-31 23:32:23,592 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:32:23,593 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:32:23,635 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:32:23,640 DEBUG HandlerThread:1765812 [system_info.py:__init__():26] System info init +2024-12-31 23:32:23,640 DEBUG HandlerThread:1765812 [system_info.py:__init__():41] System info init done +2024-12-31 23:32:23,640 INFO HandlerThread:1765812 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:32:23,640 INFO SystemMonitor:1765812 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:32:23,640 INFO HandlerThread:1765812 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:32:23,641 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:32:23,641 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:32:23,642 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:32:23,642 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:32:23,643 INFO SystemMonitor:1765812 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:32:23,653 DEBUG HandlerThread:1765812 [system_info.py:probe():152] Probing system +2024-12-31 23:32:23,655 DEBUG HandlerThread:1765812 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:probe():200] Probing system done +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:32:23.654015', 'startedAt': '2025-01-01T04:32:23.252341', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.9963749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.006, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 585.1749725341797}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:32:23,660 INFO HandlerThread:1765812 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:32:23,660 INFO HandlerThread:1765812 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:32:23,660 DEBUG HandlerThread:1765812 [system_info.py:_save_code():44] Saving code +2024-12-31 23:32:23,661 DEBUG HandlerThread:1765812 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:32:23,662 DEBUG HandlerThread:1765812 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:32:23,723 DEBUG HandlerThread:1765812 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:32:23,724 INFO HandlerThread:1765812 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:32:23,727 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:32:23,727 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:32:23,727 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:32:23,728 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:32:23,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:32:23,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:32:23,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:23,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:23,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:23,968 INFO HandlerThread:1765812 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/tb" + save: true +} +control { + relay_id: "730dfe5eef704cfb8b1e23cb2ed50623" +} +_info { + stream_id: "k7tzwekb" +} + +2024-12-31 23:32:23,968 DEBUG HandlerThread:1765812 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:32:23,992 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/ryq7cqux-wandb-metadata.json +2024-12-31 23:32:23,996 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/fhupt9xd-diff.patch +2024-12-31 23:32:24,017 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/4hiq4fyd-code/FISH/eval_robot.py +2024-12-31 23:32:24,049 DEBUG SenderThread:1765812 [sender.py:send():379] send: telemetry +2024-12-31 23:32:24,050 DEBUG SenderThread:1765812 [sender.py:send():379] send: tbrecord +2024-12-31 23:32:24,050 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:32:24,050 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 with policy live +2024-12-31 23:32:24,240 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/l1hvbsv9-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:32:24,578 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code +2024-12-31 23:32:24,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH +2024-12-31 23:32:24,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:25,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:26,578 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:26,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:27,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:28,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:28,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:29,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:30,831 DEBUG SenderThread:1765812 [sender.py:send():379] send: config +2024-12-31 23:32:30,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:31,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:32,579 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:32,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:33,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:33,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:34,580 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:34,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:35,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:36,581 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:36,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:37,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:38,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:38,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:38,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:39,023 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:39,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:40,582 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:40,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:41,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:42,582 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:42,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:43,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:44,182 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:44,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:45,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:46,583 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:32:46,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:47,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:48,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:49,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:49,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:50,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:51,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:52,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:53,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:32:53,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:32:54,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:54,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:55,028 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:32:55,586 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:32:55,971 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:56,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:57,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:58,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:32:59,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:00,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:00,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:01,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:02,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:03,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:04,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:05,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:06,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:06,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:07,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:08,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:08,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:09,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:09,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:10,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:11,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:12,112 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:12,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:13,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:14,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:15,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:16,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:17,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:17,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:18,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:19,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:20,593 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:20,671 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:33:20,671 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 with policy now +2024-12-31 23:33:20,671 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:33:20,672 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:33:20,673 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:33:20,673 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:33:20,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:21,020 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/ko0udlk1-media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos +2024-12-31 23:33:21,595 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media +2024-12-31 23:33:21,946 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/19027wpk-media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:33:21,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:22,594 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:22,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:23,643 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:33:23,644 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:33:23,644 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:23,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:23,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:24,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:24,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:25,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:26,596 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:26,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:27,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:28,597 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:28,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:28,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:29,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:30,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:31,598 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:33:31,859 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/fbp5oznv-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:33:31,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:32,598 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:32,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:33,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:34,599 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:34,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:34,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:35,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:36,599 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:36,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:37,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:38,600 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:33:38,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:38,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:39,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:39,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:40,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:40,979 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:41,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:42,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:43,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:44,980 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:45,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:45,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:46,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:47,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:48,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:49,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:50,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:50,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:51,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:52,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:53,644 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:33:53,966 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:33:53,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:33:54,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:54,981 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:55,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:56,074 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:33:56,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:57,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:58,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:33:59,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:00,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:01,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:01,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:02,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:03,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:04,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:05,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:06,970 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:06,982 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:07,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:08,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:08,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:09,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:09,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:10,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:11,985 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:12,296 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:12,540 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:34:12,570 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 with policy now +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:34:12,570 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:34:12,582 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:34:12,616 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:34:12,616 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:12,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:12,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:34:12,883 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/5avac1s8-media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:12,985 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:13,241 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/id2rq8kr-media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:34:13,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:14,617 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:14,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:15,989 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:16,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:17,521 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:17,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:18,618 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:18,989 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:19,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:20,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:34:20,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:20,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:21,378 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:34:21,378 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:34:21,378 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:34:21,620 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:34:21,994 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:22,609 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:22,621 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:22,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:23,645 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:34:23,969 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:23,969 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:24,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:24,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:25,995 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:26,622 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:26,995 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:27,652 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:27,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:28,623 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:34:28,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:29,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:30,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:31,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:32,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:32,996 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:33,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:34,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:35,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:36,997 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:37,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:38,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:38,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:39,009 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:39,083 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:39,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:40,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:41,998 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:42,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:43,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:44,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:44,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:45,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:47,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:48,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:48,999 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:49,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:50,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:51,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:52,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:53,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:53,645 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:34:53,967 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:34:53,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:34:54,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:55,000 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:55,082 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:34:56,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:57,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:58,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:34:59,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:00,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:00,976 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:01,001 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:02,005 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:02,209 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:35:02,209 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 with policy now +2024-12-31 23:35:02,213 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:35:02,214 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:02,214 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:02,221 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:02,546 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/4w0lfl6k-media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:02,634 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:02,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:35:02,947 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/f5l8nbpp-media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:35:03,013 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:04,012 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:04,635 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:05,014 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:05,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:06,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:07,020 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:08,023 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:08,636 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:08,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:08,977 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:09,017 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:10,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:10,637 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:11,022 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:11,112 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:12,020 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:12,222 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:12,222 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:12,223 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:12,638 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:12,638 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:35:12,918 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/wnuar5hj-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:35:13,022 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:14,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:14,639 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:15,025 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:16,031 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:16,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:17,028 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:18,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:18,640 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:19,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:20,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:20,641 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:21,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:21,977 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:22,033 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:23,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:23,646 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:35:23,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:23,968 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:24,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:25,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:26,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:27,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:27,124 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:28,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:29,034 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:30,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:31,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:32,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:32,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:33,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:34,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:35,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:36,035 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:37,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:37,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:38,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:38,968 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:38,968 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:39,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:40,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:41,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:42,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:43,036 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:43,104 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:44,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:45,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:46,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:47,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:48,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:48,325 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:48,650 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:49,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:50,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:51,037 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:52,038 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:53,049 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:53,365 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:53,607 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:35:53,607 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 with policy now +2024-12-31 23:35:53,612 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:35:53,612 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:35:53,612 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:35:53,634 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:35:53,646 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:53,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:35:53,904 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/b87wtpcs-media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:53,975 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:35:53,975 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:35:54,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:54,204 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/73j44g99-media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:35:54,652 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:55,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:56,040 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:57,046 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:58,044 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:35:58,609 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:35:58,654 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:35:59,043 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:00,044 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:01,050 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:02,045 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:02,655 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:03,046 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:03,886 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:04,043 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:04,289 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:04,289 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:04,289 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:04,656 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:04,656 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:05,049 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:06,056 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:06,657 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:07,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:08,056 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:08,657 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:08,973 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:08,974 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:09,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:09,129 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:10,113 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:10,658 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:11,061 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:12,061 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:12,659 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:13,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:14,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:14,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:15,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:16,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:17,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:18,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:19,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:19,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:20,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:21,062 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:22,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:23,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:23,647 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:36:23,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:23,972 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:24,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:25,048 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:25,089 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:26,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:26,664 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:27,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:28,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:29,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:30,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:30,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:31,063 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:32,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:33,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:34,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:35,064 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:35,983 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:36,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:37,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:38,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:38,972 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:38,972 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:39,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:40,065 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:41,066 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:41,965 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:42,067 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:42,669 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:43,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:44,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:45,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:46,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:46,938 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:46,938 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 with policy now +2024-12-31 23:36:46,941 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:46,941 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:46,941 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:46,957 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:46,974 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:47,068 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:47,230 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/uvyj7fkc-media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:47,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval +2024-12-31 23:36:48,004 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/crbdudzu-media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:36:48,073 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:48,671 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:49,073 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:50,075 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:51,088 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:52,083 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:52,673 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:52,933 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:53,082 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:53,647 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:36:53,990 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:36:53,990 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:36:54,084 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:54,673 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:55,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:56,100 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:56,357 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:56,357 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:56,358 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:56,674 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:56,674 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:56,942 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/uxtoxhk4-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:57,084 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:36:57,084 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:36:57,087 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:57,263 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:57,538 INFO wandb-upload_0:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_qqzkyke +2024-12-31 23:36:57,653 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:36:57,665 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval +2024-12-31 23:36:57,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table +2024-12-31 23:36:57,955 INFO SenderThread:1765812 [sender.py:send_request_log_artifact():1525] logged artifact run-k7tzwekb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwMzc5NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODU2NA==', 'latestArtifact': None}} +2024-12-31 23:36:57,955 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:57,955 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 23:36:57,956 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:36:57,956 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:57,956 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,957 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,957 DEBUG SenderThread:1765812 [sender.py:send():379] send: telemetry +2024-12-31 23:36:57,957 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,961 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,961 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,961 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,961 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,962 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,963 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,963 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,964 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,964 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,965 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,965 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,965 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,966 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,966 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,966 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,967 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:57,967 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:57,967 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:36:58,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:58,141 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/lqf7it18-media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:58,163 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/kb400lzc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:58,330 INFO wandb-upload_0:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_7f0h_k4 +2024-12-31 23:36:58,616 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/u9ug01rg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:36:58,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:36:58,675 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:send_request_log_artifact():1525] logged artifact run-k7tzwekb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwMzgwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODU2OA==', 'latestArtifact': None}} +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send():379] send: files +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json with policy now +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:36:58,789 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:36:58,789 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:36:59,043 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/g9xp0dmb-media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:36:59,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:36:59,676 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:00,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:00,676 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:01,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:02,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:02,988 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:03,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:04,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:05,077 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:06,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:07,079 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:08,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:08,978 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:37:08,978 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:37:09,078 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:09,138 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:09,558 DEBUG SenderThread:1765812 [sender.py:send():379] send: exit +2024-12-31 23:37:09,558 INFO SenderThread:1765812 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:37:09,558 INFO SenderThread:1765812 [sender.py:send_exit():588] handling runtime: 285 +2024-12-31 23:37:09,559 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:09,559 INFO SenderThread:1765812 [sender.py:send_exit():594] send defer +2024-12-31 23:37:09,559 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,559 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:37:09,559 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:37:09,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:37:09,560 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:37:09,560 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:37:09,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:37:09,560 INFO HandlerThread:1765812 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:37:09,560 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:37:09,560 DEBUG SystemMonitor:1765812 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:37:09,561 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:37:09,561 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:37:09,613 INFO HandlerThread:1765812 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:37:09,613 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,613 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:37:09,613 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:37:09,614 DEBUG SenderThread:1765812 [sender.py:send():379] send: stats +2024-12-31 23:37:09,614 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,614 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:37:09,614 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:09,614 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:37:09,614 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:37:09,614 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:09,614 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:37:09,679 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:10,680 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:15,682 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:16,354 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,355 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:37:16,355 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send():379] send: history +2024-12-31 23:37:16,355 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:37:16,356 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,356 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:37:16,356 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:16,357 DEBUG SenderThread:1765812 [sender.py:send():379] send: summary +2024-12-31 23:37:16,360 INFO SenderThread:1765812 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:37:16,360 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,361 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:37:16,361 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:16,361 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:16,361 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:37:16,361 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:16,361 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:37:16,559 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:16,683 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:18,349 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:37:18,349 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:18,349 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:18,349 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:37:18,350 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:18,350 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:37:18,350 INFO SenderThread:1765812 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:37:18,352 INFO SenderThread:1765812 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:37:18,354 INFO SenderThread:1765812 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:37:18,367 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:37:18,368 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:18,368 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:37:18,368 DEBUG SenderThread:1765812 [sender.py:send():379] send: artifact +2024-12-31 23:37:18,560 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:18,683 INFO Thread-12 :1765812 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:18,765 INFO wandb-upload_1:1765812 [upload_job.py:push():85] Skipped uploading /tmp/tmpqmi6n6n8/wandb-job.json +2024-12-31 23:37:18,766 INFO wandb-upload_0:1765812 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3oy2c3ah +2024-12-31 23:37:18,861 INFO wandb-upload_2:1765812 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf0rhrj42 +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNDA4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE5ODAyNA==', 'versionIndex': 399}}} +2024-12-31 23:37:19,593 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:37:19,593 INFO SenderThread:1765812 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:37:19,684 INFO SenderThread:1765812 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files +2024-12-31 23:37:19,684 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml config.yaml +2024-12-31 23:37:19,685 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json wandb-summary.json +2024-12-31 23:37:19,686 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt requirements.txt +2024-12-31 23:37:19,686 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:37:19,687 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:37:19,687 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log output.log +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/diff.patch diff.patch +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 media/videos/eval/4_eval_4_ece347de7301c56db235.mp4 +2024-12-31 23:37:19,690 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 media/videos/eval/0_eval_0_595f5f50e069a5ef6f72.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 media/videos/eval/1_eval_1_fa0681f94ab8a1121a68.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 media/videos/eval/2_eval_2_a8f6ca98f28c7680329f.mp4 +2024-12-31 23:37:19,691 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 media/videos/eval/3_eval_3_a59939926caab3bf8cab.mp4 +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json media/table/eval/total_success_rate_ci_6_a60f92e596343da90112.table.json +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:37:19,692 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:37:19,692 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:19,692 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:19,694 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:37:19,694 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:19,694 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:37:19,694 INFO SenderThread:1765812 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:37:19,871 INFO wandb-upload_0:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/requirements.txt +2024-12-31 23:37:19,887 INFO wandb-upload_1:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/config.yaml +2024-12-31 23:37:19,888 INFO wandb-upload_2:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/wandb-summary.json +2024-12-31 23:37:19,943 INFO wandb-upload_4:1765812 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/files/output.log +2024-12-31 23:37:20,006 INFO wandb-upload_3:1765812 [upload_job.py:push():130] Uploaded file /tmp/tmp977esz2bwandb/7vcbs7zi-events.out.tfevents.1735705943.leonmkim-ROG-Strix-G15CS-G15CS.1765571.0 +2024-12-31 23:37:20,206 INFO Thread-11 (_thread_body):1765812 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:37:20,207 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,207 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:37:20,207 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,207 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:37:20,207 INFO SenderThread:1765812 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:37:20,208 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:37:20,208 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,208 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:37:20,208 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,209 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:37:20,209 INFO SenderThread:1765812 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:37:20,266 INFO SenderThread:1765812 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:37:20,266 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:37:20,266 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,266 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:37:20,267 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:37:20,267 INFO HandlerThread:1765812 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send():379] send: final +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send():379] send: footer +2024-12-31 23:37:20,267 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: defer +2024-12-31 23:37:20,267 INFO SenderThread:1765812 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:20,268 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:37:20,268 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:37:20,269 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:37:20,269 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:37:20,270 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:37:20,271 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:20,325 DEBUG SenderThread:1765812 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:37:20,325 INFO MainThread:1765812 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:37:20,326 INFO MainThread:1765812 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:37:20,326 INFO MainThread:1765812 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:37:20,326 DEBUG HandlerThread:1765812 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:37:20,326 INFO HandlerThread:1765812 [handler.py:finish():882] shutting down handler +2024-12-31 23:37:21,269 INFO WriterThread:1765812 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb +2024-12-31 23:37:21,325 INFO SenderThread:1765812 [sender.py:finish():1615] shutting down sender +2024-12-31 23:37:21,326 INFO SenderThread:1765812 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:37:21,326 INFO SenderThread:1765812 [file_pusher.py:join():175] waiting for file pusher diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug.log b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fed0b9c1b65cf7edd5cd372061c48d23dcda7847 --- /dev/null +++ b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:32:23,254 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Configure stats pid to 1765571 +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/settings +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug.log +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/logs/debug-internal.log +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb'} +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():619] starting backend +2024-12-31 23:32:23,255 INFO MainThread:1765571 [wandb_init.py:init():623] setting up manager +2024-12-31 23:32:23,260 INFO MainThread:1765571 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:32:23,262 INFO MainThread:1765571 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:32:23,273 INFO MainThread:1765571 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:32:23,284 INFO MainThread:1765571 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:32:23,592 INFO MainThread:1765571 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:32:23,631 INFO MainThread:1765571 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:32:23,631 INFO MainThread:1765571 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:32:23,965 INFO MainThread:1765571 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:32:23,967 INFO MainThread:1765571 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:32:23,968 INFO MainThread:1765571 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/k7tzwekb/tb, True +2024-12-31 23:32:30,830 INFO MainThread:1765571 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 23:37:21,327 WARNING MsgRouterThr:1765571 [router.py:message_loop():77] message_loop has been closed diff --git a/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4e5db8f81f073aef5fcf535eec11e437ee0b069a Binary files /dev/null and b/k7tzwekb/wandb/run-20241231_233223-k7tzwekb/run-k7tzwekb.wandb differ diff --git a/l5461azr/.hydra/config.yaml b/l5461azr/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/l5461azr/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/l5461azr/.hydra/hydra.yaml b/l5461azr/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b13bef5f862ec727e9bab090a0f3730e29308a3a --- /dev/null +++ b/l5461azr/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/l5461azr + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/l5461azr/.hydra/overrides.yaml b/l5461azr/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/l5461azr/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/l5461azr/eval_policy.log b/l5461azr/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..462da326558871e703ec9bec26697c11e9aa13e1 --- /dev/null +++ b/l5461azr/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 23:59:45,723][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 23:59:45,726][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 23:59:48,835][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/yr3ixpt5/.hydra/config.yaml b/yr3ixpt5/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/yr3ixpt5/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/yr3ixpt5/.hydra/hydra.yaml b/yr3ixpt5/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fac1528eed2b4c6bd317e602801ef8ef94d225f7 --- /dev/null +++ b/yr3ixpt5/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/yr3ixpt5/.hydra/overrides.yaml b/yr3ixpt5/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/yr3ixpt5/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/yr3ixpt5/episode_rosbags/aligned_depth_to_color_K.npy b/yr3ixpt5/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yr3ixpt5/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yr3ixpt5/episode_rosbags/cam_tf_world.npy b/yr3ixpt5/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/yr3ixpt5/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/yr3ixpt5/episode_rosbags/color_K.npy b/yr3ixpt5/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yr3ixpt5/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yr3ixpt5/episode_rosbags/depth_K.npy b/yr3ixpt5/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/yr3ixpt5/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/yr3ixpt5/eval_robot.log b/yr3ixpt5/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0d76f6fdc71d59ca096544ecbbec4f4be5fe8560 --- /dev/null +++ b/yr3ixpt5/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:58:07,823][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:58:07,826][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/yr3ixpt5/eval_video/0_eval.mp4 b/yr3ixpt5/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74654e361657190f14ac7fe5704179fb364c64af --- /dev/null +++ b/yr3ixpt5/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5aff4381d38d17b95ef6d805f45659672d75598c54417d3b45ecd470bf4c87 +size 1436122 diff --git a/yr3ixpt5/eval_video/1_eval.mp4 b/yr3ixpt5/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ccd74d5c62ba1de0ff4182d3b5756c30b89132e3 --- /dev/null +++ b/yr3ixpt5/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cdb722efe029d6325c44392cff0727ca977715fe0d792ed82ac9a5c83fec75b +size 1437836 diff --git a/yr3ixpt5/eval_video/2_eval.mp4 b/yr3ixpt5/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46d9a99059a3a1141058c11f26b2a05c91ff8e9f --- /dev/null +++ b/yr3ixpt5/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7bbe5b3dd53dbcc1d91f5159aefcd61f6adc4192740d3cb945f8239f22cfd2 +size 1444161 diff --git a/yr3ixpt5/eval_video/3_eval.mp4 b/yr3ixpt5/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1dabcd11f1413deee5d1266c5e3defd8ba0a2d9e --- /dev/null +++ b/yr3ixpt5/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052982a5ecbce60047283e5a87736a0c696d51cf248608e8511bc13be9b46880 +size 1434238 diff --git a/yr3ixpt5/eval_video/4_eval.mp4 b/yr3ixpt5/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb9bafe2a29ade7881aa7fa026581132af5ad65b --- /dev/null +++ b/yr3ixpt5/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b1c47b3104c38eb2ca040f81d91612d6d8e26a7ae916a4fc0c3cba6c9340000 +size 1454609 diff --git a/yr3ixpt5/tb/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 b/yr3ixpt5/tb/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 new file mode 100644 index 0000000000000000000000000000000000000000..8697a410e161a6c2fc9c718ba90e92b2433a50f0 --- /dev/null +++ b/yr3ixpt5/tb/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f007d13066285ab7a2009dbc80d9836caef762c2d4ecbcb41e06443dcda6a262 +size 6539 diff --git a/yr3ixpt5/wandb/debug-internal.log b/yr3ixpt5/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a1c51d2a19979d2389d2360bf86a645954f62a36 --- /dev/null +++ b/yr3ixpt5/wandb/debug-internal.log @@ -0,0 +1,809 @@ +2025-01-01 11:58:13,599 INFO StreamThr :1800344 [internal.py:wandb_internal():85] W&B internal server running at pid: 1800344, started at: 2025-01-01 11:58:13.598965 +2025-01-01 11:58:13,600 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:58:13,619 INFO WriterThread:1800344 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb +2025-01-01 11:58:13,620 DEBUG SenderThread:1800344 [sender.py:send():379] send: header +2025-01-01 11:58:13,628 DEBUG SenderThread:1800344 [sender.py:send():379] send: run +2025-01-01 11:58:13,866 INFO SenderThread:1800344 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files +2025-01-01 11:58:13,867 INFO SenderThread:1800344 [sender.py:_start_run_threads():1188] run started: yr3ixpt5 with start time 1735750693.599592 +2025-01-01 11:58:13,892 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:58:13,892 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:58:13,965 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:58:13,974 DEBUG HandlerThread:1800344 [system_info.py:__init__():26] System info init +2025-01-01 11:58:13,974 DEBUG HandlerThread:1800344 [system_info.py:__init__():41] System info init done +2025-01-01 11:58:13,975 INFO HandlerThread:1800344 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:58:13,975 INFO SystemMonitor:1800344 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:58:13,975 INFO HandlerThread:1800344 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:58:13,976 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:58:13,976 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:58:13,977 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:58:13,977 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:58:13,978 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:58:13,988 DEBUG HandlerThread:1800344 [system_info.py:probe():152] Probing system +2025-01-01 11:58:13,990 DEBUG HandlerThread:1800344 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:probe():200] Probing system done +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:58:13.988644', 'startedAt': '2025-01-01T16:58:13.591614', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.4946250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.882, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.077, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 742.7409362792969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:58:13,994 INFO HandlerThread:1800344 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:58:13,994 INFO HandlerThread:1800344 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:_save_code():44] Saving code +2025-01-01 11:58:14,000 DEBUG HandlerThread:1800344 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:58:14,000 DEBUG HandlerThread:1800344 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:58:14,066 DEBUG HandlerThread:1800344 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:58:14,066 INFO HandlerThread:1800344 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:58:14,070 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:58:14,311 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:58:14,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:58:14,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:14,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:14,314 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/cey693zy-wandb-metadata.json +2025-01-01 11:58:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:14,315 INFO HandlerThread:1800344 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/tb" + save: true +} +control { + relay_id: "0ad87481909f40c89480616b0efa72ef" +} +_info { + stream_id: "yr3ixpt5" +} + +2025-01-01 11:58:14,315 DEBUG HandlerThread:1800344 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:58:14,331 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/j9miffrg-diff.patch +2025-01-01 11:58:14,348 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/1hjgwesp-code/FISH/eval_robot.py +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: telemetry +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: tbrecord +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:58:14,440 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 with policy live +2025-01-01 11:58:14,642 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/cng1qr2q-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:58:14,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json +2025-01-01 11:58:14,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code +2025-01-01 11:58:15,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:16,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:16,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:17,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:18,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:18,567 DEBUG SenderThread:1800344 [sender.py:send():379] send: config +2025-01-01 11:58:18,609 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:19,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:20,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:20,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:21,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:22,333 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:23,349 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:24,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:24,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:24,870 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:25,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:26,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:27,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:28,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:28,871 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:29,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:29,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:29,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:29,459 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:30,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:31,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:32,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:33,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:34,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:35,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:35,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:36,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:37,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:38,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:39,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:40,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:40,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:40,874 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:41,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:42,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:42,875 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:43,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:44,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:44,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:44,875 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:45,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:45,384 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:45,876 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 11:58:46,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:47,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:48,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:48,876 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:49,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:50,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:51,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:51,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:52,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:53,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:54,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:55,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:56,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:56,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:57,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:58,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:59,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:59,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:59,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:00,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:01,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:01,462 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:02,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:03,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:04,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:04,881 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:05,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:06,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:07,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:07,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:08,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:09,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:10,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:11,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:12,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:12,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:13,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:13,978 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:59:13,979 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 11:59:14,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:14,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:14,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:15,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:16,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,467 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:17,513 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:59:17,514 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:59:17,514 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 with policy now +2025-01-01 11:59:17,515 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 11:59:17,515 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:59:17,515 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:59:17,769 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/npt6mtma-media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media +2025-01-01 11:59:17,885 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 11:59:17,885 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos +2025-01-01 11:59:18,154 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/u5j370r2-media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:18,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:18,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:19,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:20,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:21,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:22,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:22,537 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:22,886 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:23,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:24,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:25,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:26,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:26,887 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:27,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:27,734 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:27,887 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:59:28,133 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/rpxl5kao-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:59:28,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:28,888 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:29,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:29,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:29,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:30,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:30,889 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:31,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:32,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:32,889 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:33,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:33,705 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:34,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:35,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:36,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:37,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:38,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:39,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:39,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:40,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:41,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:42,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,979 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 11:59:44,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:44,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:44,395 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:45,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:46,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:47,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:48,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:49,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:50,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:50,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:51,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:52,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:53,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:54,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:55,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:56,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:56,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:57,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:58,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:59,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:59,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:59,361 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:00,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:01,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:01,402 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:02,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:03,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:04,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:05,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,598 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:07,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:07,497 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:07,497 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:00:07,498 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 with policy now +2025-01-01 12:00:07,498 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:07,498 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:07,509 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:07,764 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/o3w39l1y-media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:07,900 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:07,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:07,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:00:08,159 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/q2la3smi-media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:08,337 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:08,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:09,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:10,338 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:11,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:12,338 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:12,553 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:12,902 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:13,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:13,980 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:00:14,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:14,329 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:14,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:15,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:16,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:16,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:00:16,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:17,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:17,729 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:17,735 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:17,735 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:17,736 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:17,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:18,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:18,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:19,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:20,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:21,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:22,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:22,906 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:23,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:23,703 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:24,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:25,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:26,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:26,907 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:27,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:28,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:28,908 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:29,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:29,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:29,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:29,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:30,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:31,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,909 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:33,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:34,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:34,910 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:35,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:35,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:36,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:37,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:38,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:38,911 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:39,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:40,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:40,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:41,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:42,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,980 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:00:44,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:44,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:46,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:47,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:48,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:49,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:50,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:51,317 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:51,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:52,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:53,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:54,345 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:55,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:56,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:56,556 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:56,602 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:56,603 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:00:56,603 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 with policy now +2025-01-01 12:00:56,604 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:56,604 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:56,616 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:56,863 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/d4kd8rad-media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:00:57,188 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/alrv2zie-media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:57,348 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:58,355 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:58,918 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:59,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:59,319 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:59,361 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:00,362 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:01,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:01,693 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:02,359 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:02,919 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:03,364 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:04,360 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:04,920 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:05,363 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:06,363 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:07,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:07,364 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:08,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:09,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:09,921 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:01:10,159 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/w84jc8db-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:01:10,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:10,580 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:01:10,581 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:10,581 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:10,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:01:10,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:11,366 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:12,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:12,893 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:12,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:13,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:13,981 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:01:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:14,315 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:14,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:14,923 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:15,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:16,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:17,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:18,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:18,443 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:19,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:20,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:21,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:22,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:23,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:24,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:24,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:25,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:26,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:26,927 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:27,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:28,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:29,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:29,315 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:29,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:29,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:30,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:31,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:32,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:33,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:34,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:35,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:35,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:36,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:37,375 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:38,371 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:39,374 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:40,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:40,372 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:41,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:42,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:01:44,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:44,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:44,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:45,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:45,466 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:46,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:47,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:48,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:49,376 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:49,803 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:01:49,804 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 with policy now +2025-01-01 12:01:49,806 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:01:49,807 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:01:49,807 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:49,809 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:49,935 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:01:49,935 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:49,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:01:50,058 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/yd0bdi1u-media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:50,319 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/qqtaza5z-media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:50,378 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:50,828 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:50,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:51,384 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:51,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:52,377 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:53,381 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:54,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:54,937 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:55,386 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:56,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:56,387 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:57,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:58,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:58,938 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:59,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:59,325 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:59,390 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:00,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:01,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:01,411 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:02,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:03,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:03,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:03,650 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:03,650 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:03,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:03,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:04,395 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:04,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:05,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,879 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:07,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:08,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:08,942 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:09,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:10,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:11,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:12,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:12,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:13,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:13,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:02:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:14,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:14,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:15,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:16,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:17,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:17,425 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:18,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:19,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:20,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:21,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:22,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:23,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:23,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:24,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:25,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:26,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:27,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:28,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:28,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:29,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:29,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:29,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:30,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:31,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:32,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:33,388 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:33,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:34,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:35,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:36,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:37,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:38,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:39,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:39,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:40,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:41,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:42,414 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:42,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:42,993 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:42,993 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 with policy now +2025-01-01 12:02:42,994 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:42,996 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:42,996 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:42,997 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:43,254 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/1d9wcbb0-media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:43,403 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:02:43,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:02:44,257 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/zn3udsj1-media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:44,317 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:44,318 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:44,411 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:44,444 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:44,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:45,417 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:46,423 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:47,428 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,419 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,957 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:49,422 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:49,894 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:50,421 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:50,957 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:51,422 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:52,427 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:53,427 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:54,435 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:55,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:55,439 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:56,444 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:57,440 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:57,535 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 12:02:57,536 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 12:02:57,729 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:57,943 INFO wandb-upload_1:1800344 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv8uxriq5 +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media +2025-01-01 12:02:57,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval +2025-01-01 12:02:57,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table +2025-01-01 12:02:58,142 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 12:02:58,151 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:58,338 INFO SenderThread:1800344 [sender.py:send_request_log_artifact():1525] logged artifact run-yr3ixpt5-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcxMTcwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NTU4OQ==', 'latestArtifact': None}} +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:58,339 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,342 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,342 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:58,342 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,343 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send():379] send: telemetry +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,343 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,344 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,344 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,344 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,345 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,345 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,345 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,345 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,346 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,346 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,346 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,346 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,347 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,347 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,347 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,347 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,348 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,349 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,350 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,350 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,351 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 12:02:58,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:58,519 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/o8vxjudc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:58,585 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/91bhmwzq-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:58,722 INFO wandb-upload_1:1800344 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf57gdsq7 +2025-01-01 12:02:58,787 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/i1zm59m1-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval +2025-01-01 12:02:59,104 INFO SenderThread:1800344 [sender.py:send_request_log_artifact():1525] logged artifact run-yr3ixpt5-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcxMTcxMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NTU5MA==', 'latestArtifact': None}} +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:59,105 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json with policy now +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:59,106 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:59,297 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/32y7koee-media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:59,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:59,317 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:59,431 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:59,550 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/9q64o5ua-media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:59,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:00,419 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:00,431 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:00,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:01,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:02,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:03,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:04,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:05,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,345 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:06,443 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,964 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 12:03:07,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:08,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:09,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:10,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:11,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:11,955 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:11,998 DEBUG SenderThread:1800344 [sender.py:send():379] send: exit +2025-01-01 12:03:11,998 INFO SenderThread:1800344 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 12:03:11,998 INFO SenderThread:1800344 [sender.py:send_exit():588] handling runtime: 298 +2025-01-01 12:03:11,999 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:11,999 INFO SenderThread:1800344 [sender.py:send_exit():594] send defer +2025-01-01 12:03:11,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:11,999 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 12:03:11,999 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 1 +2025-01-01 12:03:12,000 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 12:03:12,000 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 2 +2025-01-01 12:03:12,000 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 12:03:12,001 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 12:03:12,001 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 12:03:12,001 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 12:03:12,002 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined network monitor +2025-01-01 12:03:12,053 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 3 +2025-01-01 12:03:12,054 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:03:12,054 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,054 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 12:03:12,054 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 4 +2025-01-01 12:03:12,054 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,054 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 12:03:12,966 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:12,967 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:18,723 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:18,724 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:18,724 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,724 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 12:03:18,724 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 12:03:18,725 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 5 +2025-01-01 12:03:18,725 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:18,725 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,725 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 12:03:18,726 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:03:18,726 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:03:18,727 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:18,727 DEBUG SenderThread:1800344 [sender.py:send():379] send: summary +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:18,731 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 6 +2025-01-01 12:03:18,731 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,731 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 12:03:18,731 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 7 +2025-01-01 12:03:18,732 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:18,732 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,732 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 12:03:18,732 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 12:03:18,969 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:18,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:19,777 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 8 +2025-01-01 12:03:19,777 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:19,777 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,777 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 12:03:19,778 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,778 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 12:03:19,778 INFO SenderThread:1800344 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 12:03:19,780 INFO SenderThread:1800344 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 12:03:19,782 INFO SenderThread:1800344 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 12:03:19,794 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 9 +2025-01-01 12:03:19,794 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,794 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 12:03:19,795 DEBUG SenderThread:1800344 [sender.py:send():379] send: artifact +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 12:03:19,954 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 12:03:19,969 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml config.yaml +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json wandb-summary.json +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt requirements.txt +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json wandb-metadata.json +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log output.log +2025-01-01 12:03:19,972 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch diff.patch +2025-01-01 12:03:19,975 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:03:19,978 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:03:19,978 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 10 +2025-01-01 12:03:19,979 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,979 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 12:03:19,979 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,979 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 12:03:19,979 INFO SenderThread:1800344 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 12:03:19,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,000 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,151 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt +2025-01-01 12:03:20,167 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 12:03:20,180 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:20,205 INFO wandb-upload_4:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:20,256 INFO wandb-upload_3:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/yq84pazx-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:03:20,456 INFO Thread-11 (_thread_body):1800344 [sender.py:transition_state():614] send defer: 11 +2025-01-01 12:03:20,456 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,456 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 12:03:20,457 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,457 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 12:03:20,457 INFO SenderThread:1800344 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 12:03:20,458 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 12 +2025-01-01 12:03:20,458 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,458 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 12:03:20,459 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,459 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 12:03:20,459 INFO SenderThread:1800344 [file_stream.py:finish():601] file stream finish called +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [file_stream.py:finish():605] file stream finish is done +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 13 +2025-01-01 12:03:20,649 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,649 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 12:03:20,649 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 14 +2025-01-01 12:03:20,649 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,649 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: final +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: footer +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,650 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 12:03:20,650 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,651 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,651 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 12:03:20,651 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: server_info +2025-01-01 12:03:20,651 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,653 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 12:03:20,653 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 12:03:20,654 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:20,721 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,721 INFO MainThread:1800344 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 12:03:20,722 INFO MainThread:1800344 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 12:03:20,722 INFO MainThread:1800344 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 12:03:20,722 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 12:03:20,722 INFO HandlerThread:1800344 [handler.py:finish():882] shutting down handler +2025-01-01 12:03:21,653 INFO WriterThread:1800344 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb +2025-01-01 12:03:21,721 INFO SenderThread:1800344 [sender.py:finish():1615] shutting down sender +2025-01-01 12:03:21,721 INFO SenderThread:1800344 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 12:03:21,722 INFO SenderThread:1800344 [file_pusher.py:join():175] waiting for file pusher diff --git a/yr3ixpt5/wandb/debug.log b/yr3ixpt5/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dc310cc8704cf96b85398ea44b9d4ba687a9ed44 --- /dev/null +++ b/yr3ixpt5/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Configure stats pid to 1800171 +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/settings +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug.log +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug-internal.log +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5'} +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():619] starting backend +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():623] setting up manager +2025-01-01 11:58:13,598 INFO MainThread:1800171 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:58:13,599 INFO MainThread:1800171 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:58:13,619 INFO MainThread:1800171 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:58:13,626 INFO MainThread:1800171 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:58:13,891 INFO MainThread:1800171 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:58:13,960 INFO MainThread:1800171 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:58:13,961 INFO MainThread:1800171 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:58:14,312 INFO MainThread:1800171 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:58:14,312 INFO MainThread:1800171 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:58:14,313 INFO MainThread:1800171 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:58:14,313 INFO MainThread:1800171 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:58:14,314 INFO MainThread:1800171 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:58:14,314 INFO MainThread:1800171 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/tb, True +2025-01-01 11:58:18,567 INFO MainThread:1800171 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 12:03:21,723 WARNING MsgRouterThr:1800171 [router.py:message_loop():77] message_loop has been closed diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a971b28fad332173435f045492b5f14ddf9c42d8 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735750693 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json new file mode 100644 index 0000000000000000000000000000000000000000..05f5ec969e478e03adacf0e9ea34373445255675 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[35, 60, 0.5833333333333334, 0.4304076731344172, 0.7243532537965295, 0.45945706964349986, 0.7086090892136112, 0.4792761579197369, 0.6816811859244335, 0.49539723619910736, 0.6682300753555007]]} \ No newline at end of file diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74654e361657190f14ac7fe5704179fb364c64af --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5aff4381d38d17b95ef6d805f45659672d75598c54417d3b45ecd470bf4c87 +size 1436122 diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ccd74d5c62ba1de0ff4182d3b5756c30b89132e3 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cdb722efe029d6325c44392cff0727ca977715fe0d792ed82ac9a5c83fec75b +size 1437836 diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46d9a99059a3a1141058c11f26b2a05c91ff8e9f --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7bbe5b3dd53dbcc1d91f5159aefcd61f6adc4192740d3cb945f8239f22cfd2 +size 1444161 diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1dabcd11f1413deee5d1266c5e3defd8ba0a2d9e --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052982a5ecbce60047283e5a87736a0c696d51cf248608e8511bc13be9b46880 +size 1434238 diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb9bafe2a29ade7881aa7fa026581132af5ad65b --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b1c47b3104c38eb2ca040f81d91612d6d8e26a7ae916a4fc0c3cba6c9340000 +size 1454609 diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..07dfc5738fbcce583c297242cb75025be38694f0 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735750698.568400]: resetting environment +[INFO] [1735750698.576225]: cleared current plan +[INFO] [1735750698.578347]: moving to home +[INFO] [1735750699.581052]: reached home +[INFO] [1735750699.582830]: reset action history +[INFO] [1735750702.292107]: environment reset +Starting episode 0 +[INFO] [1735750702.297477]: resetting environment +[INFO] [1735750702.304202]: cleared current plan +[INFO] [1735750702.305467]: moving to home +[INFO] [1735750703.307647]: reached home +[INFO] [1735750703.309199]: reset action history +[INFO] [1735750706.019238]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735750718.007098]: resetting environment +[INFO] [1735750718.010909]: cleared current plan +[INFO] [1735750718.011950]: moving to home +[INFO] [1735750719.013260]: reached home +[INFO] [1735750719.014500]: reset action history +[INFO] [1735750721.724039]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/episode_rosbags/episode_0_2025-01-01-11-58-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1800171.log +started roslaunch server http://158.130.50.37:33067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1800450] +process[print_text-2]: started with pid [1800474] +process[rosbag_record-3]: started with pid [1800475] +started bagging! +For topic gripper_width: timestamp difference is 106112888 for nearest: 1735750726445300272 - target: 1735750726339187384 at idx 59 +For topic gripper_width: timestamp difference is -110346230 for nearest: 1735750742878623573 - target: 1735750742988969803 at idx 249 +[INFO] [1735750756.542491]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735750757.266430]: Stored episode 1. +[INFO] [1735750757.266709]: Saving video... +[INFO] [1735750757.467069]: Video saved! +Starting episode 1 +[INFO] [1735750757.478797]: resetting environment +[INFO] [1735750757.488800]: cleared current plan +[INFO] [1735750757.489105]: moving to home +[INFO] [1735750761.491767]: reached home +[INFO] [1735750761.492483]: reset action history +[INFO] [1735750764.201393]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750767.738640]: resetting environment +[INFO] [1735750767.745583]: cleared current plan +[INFO] [1735750767.746073]: moving to home +[INFO] [1735750768.747636]: reached home +[INFO] [1735750768.748194]: reset action history +[INFO] [1735750771.455447]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/episode_rosbags/episode_1_2025-01-01-11-59-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1800171.log +started roslaunch server http://158.130.50.37:36761/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1800599] +process[print_text-5]: started with pid [1800600] +process[rosbag_record-6]: started with pid [1800601] +started bagging! +[INFO] [1735750806.194137]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735750806.994040]: Stored episode 2. +[INFO] [1735750806.994297]: Saving video... +[INFO] [1735750807.452120]: Video saved! +Starting episode 2 +[INFO] [1735750807.483961]: resetting environment +[INFO] [1735750807.506244]: cleared current plan +[INFO] [1735750807.506628]: moving to home +[INFO] [1735750811.510959]: reached home +[INFO] [1735750811.514483]: reset action history +[INFO] [1735750814.288827]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750816.695524]: resetting environment +[INFO] [1735750816.713513]: cleared current plan +[INFO] [1735750816.713792]: moving to home +[INFO] [1735750817.715434]: reached home +[INFO] [1735750817.716448]: reset action history +[INFO] [1735750820.423414]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/episode_rosbags/episode_2_2025-01-01-12-00-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1800171.log +started roslaunch server http://158.130.50.37:40719/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1800741] +process[print_text-8]: started with pid [1800765] +process[rosbag_record-9]: started with pid [1800766] +started bagging! +For topic gripper_width: timestamp difference is 102553377 for nearest: 1735750825045223007 - target: 1735750824942669630 at idx 64 +For topic gripper_width: timestamp difference is 103021345 for nearest: 1735750826345270595 - target: 1735750826242249250 at idx 77 +For topic gripper_width: timestamp difference is 102376241 for nearest: 1735750831045270461 - target: 1735750830942894220 at idx 139 +For topic gripper_width: timestamp difference is 104089849 for nearest: 1735750833145295732 - target: 1735750833041205883 at idx 162 +For topic gripper_width: timestamp difference is 104473538 for nearest: 1735750833645270772 - target: 1735750833540797234 at idx 169 +For topic gripper_width: timestamp difference is 106573042 for nearest: 1735750836245272812 - target: 1735750836138699770 at idx 201 +[INFO] [1735750855.344522]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735750856.128762]: Stored episode 3. +[INFO] [1735750856.134273]: Saving video... +[INFO] [1735750856.555652]: Video saved! +Starting episode 3 +[INFO] [1735750856.616430]: resetting environment +[INFO] [1735750856.642382]: cleared current plan +[INFO] [1735750856.643686]: moving to home +[INFO] [1735750860.647508]: reached home +[INFO] [1735750860.653770]: reset action history +[INFO] [1735750863.447170]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750869.803664]: resetting environment +[INFO] [1735750869.840008]: cleared current plan +[INFO] [1735750869.844313]: moving to home +[INFO] [1735750870.851566]: reached home +[INFO] [1735750870.853423]: reset action history +[INFO] [1735750873.586470]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/episode_rosbags/episode_3_2025-01-01-12-01-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1800171.log +started roslaunch server http://158.130.50.37:41369/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1800895] +process[print_text-11]: started with pid [1800896] +process[rosbag_record-12]: started with pid [1800920] +started bagging! +For topic gripper_width: timestamp difference is 111350694 for nearest: 1735750884445301929 - target: 1735750884333951235 at idx 153 +[INFO] [1735750908.348767]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735750909.108835]: Stored episode 4. +[INFO] [1735750909.112926]: Saving video... +[INFO] [1735750909.741079]: Video saved! +Starting episode 4 +[INFO] [1735750909.809241]: resetting environment +[INFO] [1735750909.826360]: cleared current plan +[INFO] [1735750909.827569]: moving to home +[WARN] [1735750909.690338]: Plan exhausted +[WARN] [1735750909.730254]: Plan exhausted +[WARN] [1735750909.770255]: Plan exhausted +[WARN] [1735750909.811746]: Plan exhausted +[INFO] [1735750913.837051]: reached home +[INFO] [1735750913.840401]: reset action history +[INFO] [1735750916.621160]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750923.055177]: resetting environment +[INFO] [1735750923.075678]: cleared current plan +[INFO] [1735750923.079581]: moving to home +[INFO] [1735750924.089243]: reached home +[INFO] [1735750924.093043]: reset action history +[INFO] [1735750926.877655]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/episode_rosbags/episode_4_2025-01-01-12-02-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1800171.log +started roslaunch server http://158.130.50.37:33647/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1801080] +process[print_text-14]: started with pid [1801081] +process[rosbag_record-15]: started with pid [1801082] +started bagging! +[INFO] [1735750961.846084]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1735750962.665322]: Stored episode 5. +[INFO] [1735750962.677291]: Saving video... +[INFO] [1735750962.925862]: Video saved! +[INFO] [1735750962.997096]: resetting environment +[INFO] [1735750963.031630]: cleared current plan +[INFO] [1735750963.035330]: moving to home +[INFO] [1735750967.038725]: reached home +[INFO] [1735750967.046417]: reset action history +[INFO] [1735750969.893155]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2fd836df3bfff72f4b1c11d722bae6b73a205491 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:58:13.988644", + "startedAt": "2025-01-01T16:58:13.591614", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.4946250000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.882, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.077, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 742.7409362792969 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6a95b92de6dee57735ead5afd7536acf953025a8 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6e5aff4381d38d17b95ef6d805f45659672d75598c54417d3b45ecd470bf4c87", "size": 1436122, "path": "media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4"}, "global_step": 665, "_timestamp": 1735750978.045921, "_runtime": 284.4463291168213, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2cdb722efe029d6325c44392cff0727ca977715fe0d792ed82ac9a5c83fec75b", "size": 1437836, "path": "media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4"}, "eval/num_success": 5.0, "episode": 60.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "3a7bbe5b3dd53dbcc1d91f5159aefcd61f6adc4192740d3cb945f8239f22cfd2", "size": 1444161, "path": "media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "052982a5ecbce60047283e5a87736a0c696d51cf248608e8511bc13be9b46880", "size": 1434238, "path": "media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1b1c47b3104c38eb2ca040f81d91612d6d8e26a7ae916a4fc0c3cba6c9340000", "size": 1454609, "path": "media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://5mjxyeanbdpzig9vfep4e3afnpqp7286l34prkqgy08ejmahocmtwa6r4z71rlsjhex5h9jkavjrvtki2orr9dsionmxh8qsjaqcyq954mzsxcj03wajwifeafxib71w/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8947ldqnmdl8pxdtycmwuskc5emmlbk3i6nznqubxf7icwb2yy3shsaojbxtnzj8sfxmusmru27xb7bvqmr9jyo05thaqh8010lbejmtyizo4dhafynz8liaknwtn3xc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 35, "total_num_episodes": 60, "total_success_rate": 0.5833333333333334, "total_success_rate_lb_0.01": 0.4304076731344172, "total_success_rate_ub_0.01": 0.7243532537965295, "total_success_rate_lb_0.025": 0.45945706964349986, "total_success_rate_ub_0.025": 0.7086090892136112, "total_success_rate_lb_0.05": 0.4792761579197369, "total_success_rate_ub_0.05": 0.6816811859244335, "total_success_rate_lb_0.1": 0.49539723619910736, "total_success_rate_ub_0.1": 0.6682300753555007, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2aa01f31ff0b696b0c62c0ed286abf30ffac71ba61e1f8a27c2ef847b012a96e", "size": 522, "artifact_path": "wandb-client-artifact://gmg6gwmb1fxxk3s2ug7uypjsz47djiwyahc6vecd53qq284zxksox24rw2v591zawb84fbw2ig0vrtg1nky3sv7s6hgjzaaelubr8j1sbgzfxc9rii2baai3hyxupg04/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://dxcct0duikjx7ggjy94jeaslq8zmc7k6ub49d5ppk5r6abdhvhe490fs6cpficu6edti9zo1o2sjz52u02wwe8zm7789nccs4sxlyenqh8hszsefcfv5ibivgtj4n633:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 298}, "eval/total_success_rate": 0.5833333134651184, "eval/total_success_rate_lb_0.01": 0.43040767312049866, "eval/total_success_rate_ub_0.01": 0.7243532538414001, "eval/total_success_rate_lb_0.025": 0.45945706963539124, "eval/total_success_rate_ub_0.025": 0.7086091041564941, "eval/total_success_rate_lb_0.05": 0.4792761504650116, "eval/total_success_rate_ub_0.05": 0.6816811561584473, "eval/total_success_rate_lb_0.1": 0.4953972399234772, "eval/total_success_rate_ub_0.1": 0.6682300567626953} \ No newline at end of file diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug-internal.log b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a1c51d2a19979d2389d2360bf86a645954f62a36 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug-internal.log @@ -0,0 +1,809 @@ +2025-01-01 11:58:13,599 INFO StreamThr :1800344 [internal.py:wandb_internal():85] W&B internal server running at pid: 1800344, started at: 2025-01-01 11:58:13.598965 +2025-01-01 11:58:13,600 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:58:13,619 INFO WriterThread:1800344 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb +2025-01-01 11:58:13,620 DEBUG SenderThread:1800344 [sender.py:send():379] send: header +2025-01-01 11:58:13,628 DEBUG SenderThread:1800344 [sender.py:send():379] send: run +2025-01-01 11:58:13,866 INFO SenderThread:1800344 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files +2025-01-01 11:58:13,867 INFO SenderThread:1800344 [sender.py:_start_run_threads():1188] run started: yr3ixpt5 with start time 1735750693.599592 +2025-01-01 11:58:13,892 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:58:13,892 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:58:13,965 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:58:13,974 DEBUG HandlerThread:1800344 [system_info.py:__init__():26] System info init +2025-01-01 11:58:13,974 DEBUG HandlerThread:1800344 [system_info.py:__init__():41] System info init done +2025-01-01 11:58:13,975 INFO HandlerThread:1800344 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:58:13,975 INFO SystemMonitor:1800344 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:58:13,975 INFO HandlerThread:1800344 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:58:13,976 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:58:13,976 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:58:13,977 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:58:13,977 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:58:13,978 INFO SystemMonitor:1800344 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:58:13,988 DEBUG HandlerThread:1800344 [system_info.py:probe():152] Probing system +2025-01-01 11:58:13,990 DEBUG HandlerThread:1800344 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:probe():200] Probing system done +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:58:13.988644', 'startedAt': '2025-01-01T16:58:13.591614', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.4946250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.882, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.077, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 742.7409362792969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:58:13,994 INFO HandlerThread:1800344 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:58:13,994 INFO HandlerThread:1800344 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:58:13,994 DEBUG HandlerThread:1800344 [system_info.py:_save_code():44] Saving code +2025-01-01 11:58:14,000 DEBUG HandlerThread:1800344 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:58:14,000 DEBUG HandlerThread:1800344 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:58:14,066 DEBUG HandlerThread:1800344 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:58:14,066 INFO HandlerThread:1800344 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:58:14,070 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:58:14,070 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:58:14,311 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:58:14,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:58:14,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:14,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:14,314 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/cey693zy-wandb-metadata.json +2025-01-01 11:58:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:14,315 INFO HandlerThread:1800344 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/tb" + save: true +} +control { + relay_id: "0ad87481909f40c89480616b0efa72ef" +} +_info { + stream_id: "yr3ixpt5" +} + +2025-01-01 11:58:14,315 DEBUG HandlerThread:1800344 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:58:14,331 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/j9miffrg-diff.patch +2025-01-01 11:58:14,348 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/1hjgwesp-code/FISH/eval_robot.py +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: telemetry +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: tbrecord +2025-01-01 11:58:14,440 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:58:14,440 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 with policy live +2025-01-01 11:58:14,642 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/cng1qr2q-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:58:14,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json +2025-01-01 11:58:14,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH +2025-01-01 11:58:14,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code +2025-01-01 11:58:15,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:16,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:16,868 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:17,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:18,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:18,567 DEBUG SenderThread:1800344 [sender.py:send():379] send: config +2025-01-01 11:58:18,609 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:19,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:20,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:20,869 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:21,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:22,333 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:23,349 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:24,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:24,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:24,870 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:25,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:26,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:27,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:28,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:28,871 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:29,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:29,312 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:29,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:29,459 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:30,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:31,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:32,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:33,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:34,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:35,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:35,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:36,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:37,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:38,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:39,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:40,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:40,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:40,874 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:41,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:42,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:42,875 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:43,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:44,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:44,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:44,875 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:45,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:45,384 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:45,876 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 11:58:46,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:47,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:48,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:48,876 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:58:49,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:50,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:51,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:51,319 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:52,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:53,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:54,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:55,320 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:56,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:58:56,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:57,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:58,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:58:59,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:58:59,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:58:59,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:00,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:01,321 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:01,462 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:02,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:03,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:04,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:04,881 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:05,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:06,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:07,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:07,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:08,322 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:09,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:10,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:11,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:12,314 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:12,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:13,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:13,978 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:59:13,979 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 11:59:14,312 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:14,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:14,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:15,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:16,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:17,467 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:17,513 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:59:17,514 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 11:59:17,514 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 with policy now +2025-01-01 11:59:17,515 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 11:59:17,515 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:59:17,515 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:59:17,769 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/npt6mtma-media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 11:59:17,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media +2025-01-01 11:59:17,885 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 11:59:17,885 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos +2025-01-01 11:59:18,154 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/u5j370r2-media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 11:59:18,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:18,884 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:19,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:20,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:21,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:22,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:22,537 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:22,886 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:23,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:24,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:25,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:26,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:26,887 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:27,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:27,734 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:27,887 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:59:28,133 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/rpxl5kao-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 11:59:28,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:28,888 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:29,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:29,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:29,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:30,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:30,889 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:31,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:32,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:32,889 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 11:59:33,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:33,705 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:34,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:35,326 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:36,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:37,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:38,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:39,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:39,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:40,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:41,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:42,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:43,979 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 11:59:44,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:44,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:44,395 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:45,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:46,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:47,327 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:48,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:49,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:50,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:50,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:51,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:52,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:53,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:54,328 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:55,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:56,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:59:56,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:57,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:58,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:59:59,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:59:59,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:59:59,361 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:00,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:01,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:01,402 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:02,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:03,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:04,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:05,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,330 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:06,598 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:07,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:07,497 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:07,497 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:00:07,498 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 with policy now +2025-01-01 12:00:07,498 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:07,498 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:07,509 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:07,764 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/o3w39l1y-media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:07,900 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:07,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:07,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:00:08,159 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/q2la3smi-media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:00:08,337 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:08,901 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:09,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:10,338 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:11,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:12,338 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:12,553 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:12,902 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:13,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:13,980 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:00:14,329 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:14,329 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:14,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:15,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:16,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:16,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:00:16,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:17,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:17,729 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:17,735 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:17,735 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:17,736 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:17,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:18,339 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:18,904 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:19,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:20,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:21,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:22,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:22,906 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:23,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:23,703 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:24,340 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:25,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:26,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:26,907 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:27,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:28,341 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:28,908 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:29,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:29,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:29,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:29,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:30,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:31,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:32,909 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:33,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:34,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:34,910 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:35,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:35,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:36,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:37,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:38,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:38,911 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:39,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:40,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:40,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:41,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:42,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,343 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:43,980 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:00:44,313 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:44,313 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:44,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:45,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:46,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:47,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:48,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:49,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:50,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:51,317 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:51,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:52,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:53,344 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:54,345 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:55,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:56,353 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:56,556 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:00:56,602 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:00:56,603 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:00:56,603 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 with policy now +2025-01-01 12:00:56,604 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:00:56,604 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:00:56,616 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:00:56,863 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/d4kd8rad-media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:56,917 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:00:57,188 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/alrv2zie-media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:00:57,348 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:58,355 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:00:58,918 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:00:59,318 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:00:59,319 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:00:59,361 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:00,362 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:01,357 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:01,693 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:02,359 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:02,919 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:03,364 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:04,360 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:04,920 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:05,363 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:06,363 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:07,324 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:07,364 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:08,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:09,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:09,921 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:01:10,159 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/w84jc8db-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:01:10,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:10,580 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:01:10,581 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:10,581 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:10,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:01:10,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:11,366 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:12,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:12,893 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:12,922 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:13,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:13,981 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:01:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:14,315 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:14,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:14,923 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:15,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:16,367 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:17,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:18,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:18,443 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:19,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:20,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:21,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:22,368 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:23,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:24,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:24,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:25,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:26,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:26,927 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:27,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:28,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:29,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:29,315 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:29,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:29,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:30,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:31,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:32,369 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:33,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:34,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:35,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:35,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:36,370 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:37,375 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:38,371 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:39,374 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:40,323 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:40,372 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:41,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:42,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:43,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:01:44,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:44,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:44,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:45,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:45,466 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:46,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:47,373 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:48,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:49,376 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:49,803 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:01:49,804 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 with policy now +2025-01-01 12:01:49,806 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:01:49,807 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:01:49,807 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:01:49,809 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:01:49,935 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:01:49,935 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:49,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:01:50,058 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/yd0bdi1u-media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:50,319 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/qqtaza5z-media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:01:50,378 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:50,828 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:50,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:51,384 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:51,936 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:52,377 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:53,381 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:54,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:54,937 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:55,386 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:56,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:01:56,387 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:57,389 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:58,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:01:58,938 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:01:59,325 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:01:59,325 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:01:59,390 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:00,392 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:01,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:01,411 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:02,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:03,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:03,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:03,650 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:03,650 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:03,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:03,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:04,395 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:04,940 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:05,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:06,879 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:07,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:08,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:08,942 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:09,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:10,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:11,393 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:12,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:12,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:13,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:13,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:02:14,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:14,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:14,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:15,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:16,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:17,394 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:17,425 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:18,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:19,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:20,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:21,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:22,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:23,334 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:23,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:24,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:25,396 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:26,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:27,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:28,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:28,397 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:29,315 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:29,316 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:29,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:30,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:31,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:32,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:33,388 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:33,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:34,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:35,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:36,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:37,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:38,398 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:39,335 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:39,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:40,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:41,399 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:42,414 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:42,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:42,993 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:42,993 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 with policy now +2025-01-01 12:02:42,994 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:42,996 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:42,996 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:42,997 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:43,254 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/1d9wcbb0-media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:43,403 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:43,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval +2025-01-01 12:02:43,982 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:02:44,257 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/zn3udsj1-media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:02:44,317 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:44,318 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:44,411 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:44,444 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:44,955 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:45,417 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:46,423 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:47,428 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,419 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:48,957 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:49,422 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:49,894 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:50,421 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:50,957 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:51,422 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:52,427 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:53,427 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:54,435 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:55,342 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:02:55,439 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:56,444 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:57,440 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:57,535 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 12:02:57,536 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 12:02:57,729 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:57,943 INFO wandb-upload_1:1800344 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv8uxriq5 +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:57,960 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media +2025-01-01 12:02:57,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval +2025-01-01 12:02:57,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table +2025-01-01 12:02:58,142 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 12:02:58,151 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 12:02:58,338 INFO SenderThread:1800344 [sender.py:send_request_log_artifact():1525] logged artifact run-yr3ixpt5-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcxMTcwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NTU4OQ==', 'latestArtifact': None}} +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:58,339 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:58,339 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,342 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,342 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:58,342 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,343 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send():379] send: telemetry +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,343 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,343 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,344 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,344 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,344 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,345 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,345 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,345 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,345 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,346 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,346 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,346 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,346 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,347 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,347 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,347 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,347 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,348 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,349 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,350 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:58,350 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:58,351 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 12:02:58,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:58,519 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/o8vxjudc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:58,585 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/91bhmwzq-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:58,722 INFO wandb-upload_1:1800344 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf57gdsq7 +2025-01-01 12:02:58,787 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/i1zm59m1-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:58,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval +2025-01-01 12:02:59,104 INFO SenderThread:1800344 [sender.py:send_request_log_artifact():1525] logged artifact run-yr3ixpt5-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcxMTcxMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NTU5MA==', 'latestArtifact': None}} +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send():379] send: files +2025-01-01 12:02:59,105 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json with policy now +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:02:59,105 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:02:59,106 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:02:59,297 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/32y7koee-media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:59,316 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 12:02:59,317 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: stop_status +2025-01-01 12:02:59,431 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:02:59,550 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/9q64o5ua-media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:02:59,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:00,419 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:00,431 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:00,961 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:01,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:02,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:03,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:04,432 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:05,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,345 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:06,443 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:06,964 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 12:03:07,433 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:08,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:09,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:10,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:11,434 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:11,955 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:11,998 DEBUG SenderThread:1800344 [sender.py:send():379] send: exit +2025-01-01 12:03:11,998 INFO SenderThread:1800344 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 12:03:11,998 INFO SenderThread:1800344 [sender.py:send_exit():588] handling runtime: 298 +2025-01-01 12:03:11,999 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:11,999 INFO SenderThread:1800344 [sender.py:send_exit():594] send defer +2025-01-01 12:03:11,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:11,999 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 12:03:11,999 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 1 +2025-01-01 12:03:12,000 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 12:03:12,000 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 12:03:12,000 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 2 +2025-01-01 12:03:12,000 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 12:03:12,000 INFO HandlerThread:1800344 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 12:03:12,001 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 12:03:12,001 DEBUG SystemMonitor:1800344 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 12:03:12,001 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 12:03:12,002 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 12:03:12,053 INFO HandlerThread:1800344 [interfaces.py:finish():200] Joined network monitor +2025-01-01 12:03:12,053 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 3 +2025-01-01 12:03:12,054 DEBUG SenderThread:1800344 [sender.py:send():379] send: stats +2025-01-01 12:03:12,054 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,054 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 12:03:12,054 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 12:03:12,054 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 4 +2025-01-01 12:03:12,054 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:12,054 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 12:03:12,966 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:12,967 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:18,723 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:18,724 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:18,724 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,724 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 12:03:18,724 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 12:03:18,725 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 5 +2025-01-01 12:03:18,725 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:18,725 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,725 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 12:03:18,726 DEBUG SenderThread:1800344 [sender.py:send():379] send: history +2025-01-01 12:03:18,726 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: summary_record +2025-01-01 12:03:18,727 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:18,727 DEBUG SenderThread:1800344 [sender.py:send():379] send: summary +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 12:03:18,731 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 12:03:18,731 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 6 +2025-01-01 12:03:18,731 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,731 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 12:03:18,731 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 7 +2025-01-01 12:03:18,732 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 12:03:18,732 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:18,732 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 12:03:18,732 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:18,732 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 12:03:18,969 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:18,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:19,777 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 8 +2025-01-01 12:03:19,777 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:19,777 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,777 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 12:03:19,778 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,778 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 12:03:19,778 INFO SenderThread:1800344 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 12:03:19,780 INFO SenderThread:1800344 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 12:03:19,782 INFO SenderThread:1800344 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 12:03:19,794 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 9 +2025-01-01 12:03:19,794 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,794 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 12:03:19,795 DEBUG SenderThread:1800344 [sender.py:send():379] send: artifact +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 12:03:19,954 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 12:03:19,954 INFO SenderThread:1800344 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 12:03:19,969 INFO Thread-12 :1800344 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml config.yaml +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json wandb-summary.json +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt requirements.txt +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-metadata.json wandb-metadata.json +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:03:19,970 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log output.log +2025-01-01 12:03:19,972 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/diff.patch diff.patch +2025-01-01 12:03:19,975 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 media/videos/eval/1_eval_1_2cdb722efe029d6325c4.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 media/videos/eval/2_eval_2_3a7bbe5b3dd53dbcc1d9.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 media/videos/eval/0_eval_0_6e5aff4381d38d17b95e.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 media/videos/eval/4_eval_4_1b1c47b3104c38eb2ca0.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 media/videos/eval/3_eval_3_052982a5ecbce6004728.mp4 +2025-01-01 12:03:19,976 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json media/table/eval/total_success_rate_ci_6_2aa01f31ff0b696b0c62.table.json +2025-01-01 12:03:19,978 INFO SenderThread:1800344 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 12:03:19,978 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 10 +2025-01-01 12:03:19,979 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:19,979 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 12:03:19,979 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:19,979 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 12:03:19,979 INFO SenderThread:1800344 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 12:03:19,999 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,000 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,151 INFO wandb-upload_2:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/requirements.txt +2025-01-01 12:03:20,167 INFO wandb-upload_0:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/config.yaml +2025-01-01 12:03:20,180 INFO wandb-upload_1:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/wandb-summary.json +2025-01-01 12:03:20,205 INFO wandb-upload_4:1800344 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/files/output.log +2025-01-01 12:03:20,256 INFO wandb-upload_3:1800344 [upload_job.py:push():130] Uploaded file /tmp/tmplyxvj7c2wandb/yq84pazx-events.out.tfevents.1735750694.leonmkim-ROG-Strix-G15CS-G15CS.1800171.0 +2025-01-01 12:03:20,456 INFO Thread-11 (_thread_body):1800344 [sender.py:transition_state():614] send defer: 11 +2025-01-01 12:03:20,456 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,456 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 12:03:20,457 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,457 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 12:03:20,457 INFO SenderThread:1800344 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 12:03:20,458 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 12 +2025-01-01 12:03:20,458 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,458 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 12:03:20,459 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,459 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 12:03:20,459 INFO SenderThread:1800344 [file_stream.py:finish():601] file stream finish called +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [file_stream.py:finish():605] file stream finish is done +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 13 +2025-01-01 12:03:20,649 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,649 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 12:03:20,649 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 12:03:20,649 INFO SenderThread:1800344 [sender.py:transition_state():614] send defer: 14 +2025-01-01 12:03:20,649 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: defer +2025-01-01 12:03:20,649 INFO HandlerThread:1800344 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: final +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send():379] send: footer +2025-01-01 12:03:20,650 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: defer +2025-01-01 12:03:20,650 INFO SenderThread:1800344 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 12:03:20,650 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,651 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,651 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 12:03:20,651 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: server_info +2025-01-01 12:03:20,651 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 12:03:20,653 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 12:03:20,653 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 12:03:20,654 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 12:03:20,721 DEBUG SenderThread:1800344 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 12:03:20,721 INFO MainThread:1800344 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 12:03:20,722 INFO MainThread:1800344 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 12:03:20,722 INFO MainThread:1800344 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 12:03:20,722 DEBUG HandlerThread:1800344 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 12:03:20,722 INFO HandlerThread:1800344 [handler.py:finish():882] shutting down handler +2025-01-01 12:03:21,653 INFO WriterThread:1800344 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb +2025-01-01 12:03:21,721 INFO SenderThread:1800344 [sender.py:finish():1615] shutting down sender +2025-01-01 12:03:21,721 INFO SenderThread:1800344 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 12:03:21,722 INFO SenderThread:1800344 [file_pusher.py:join():175] waiting for file pusher diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug.log b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dc310cc8704cf96b85398ea44b9d4ba687a9ed44 --- /dev/null +++ b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Configure stats pid to 1800171 +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/settings +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:58:13,593 INFO MainThread:1800171 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug.log +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/logs/debug-internal.log +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5'} +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():619] starting backend +2025-01-01 11:58:13,594 INFO MainThread:1800171 [wandb_init.py:init():623] setting up manager +2025-01-01 11:58:13,598 INFO MainThread:1800171 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:58:13,599 INFO MainThread:1800171 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:58:13,619 INFO MainThread:1800171 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:58:13,626 INFO MainThread:1800171 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:58:13,891 INFO MainThread:1800171 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:58:13,960 INFO MainThread:1800171 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:58:13,961 INFO MainThread:1800171 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:58:14,312 INFO MainThread:1800171 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:58:14,312 INFO MainThread:1800171 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:58:14,313 INFO MainThread:1800171 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:58:14,313 INFO MainThread:1800171 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:58:14,314 INFO MainThread:1800171 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:58:14,314 INFO MainThread:1800171 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/yr3ixpt5/tb, True +2025-01-01 11:58:18,567 INFO MainThread:1800171 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 12:03:21,723 WARNING MsgRouterThr:1800171 [router.py:message_loop():77] message_loop has been closed diff --git a/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7e85facbad3f427d229b69b5d0b83a65185b2b32 Binary files /dev/null and b/yr3ixpt5/wandb/run-20250101_115813-yr3ixpt5/run-yr3ixpt5.wandb differ diff --git a/z0y7vp39/episode_rosbags/episode_3_2024-12-31-21-31-12.bag b/z0y7vp39/episode_rosbags/episode_3_2024-12-31-21-31-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..8f16e4e12d7f1e4912f824bf5e9c49f4306f58b7 --- /dev/null +++ b/z0y7vp39/episode_rosbags/episode_3_2024-12-31-21-31-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d9feff090de05bf99f1a248e01e1fe07089f113b436c403266f7df0a859ff58 +size 2609377942 diff --git a/z0y7vp39/episode_rosbags/episode_4_2024-12-31-21-33-47.bag b/z0y7vp39/episode_rosbags/episode_4_2024-12-31-21-33-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..2d33a818a48e9ee8f729583935894da8b9580064 --- /dev/null +++ b/z0y7vp39/episode_rosbags/episode_4_2024-12-31-21-33-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e8c446fa6c9b4abdc7eda6e88084d150442001c00f26ecdbc5a710d89202d6 +size 2600525019