diff --git a/.gitattributes b/.gitattributes index 6992d1d306f6a01cd2a4ed4f13d9d3aa17a11762..539dd1c0cca87574b556e5ba296f7c1aaf306525 100644 --- a/.gitattributes +++ b/.gitattributes @@ -64,3 +64,52 @@ hcp6xxlt/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text hcp6xxlt/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text hcp6xxlt/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag filter=lfs diff=lfs merge=lfs -text +231uxqco/episode_rosbags/episode_1_2025-01-21-21-03-49.bag filter=lfs diff=lfs merge=lfs -text +5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/episode_rosbags/episode_0_2025-01-21-21-03-19.bag filter=lfs diff=lfs merge=lfs -text +37t5ymwr/episode_rosbags/episode_4_2025-01-21-20-59-22.bag filter=lfs diff=lfs merge=lfs -text +231uxqco/episode_rosbags/episode_4_2025-01-21-21-05-35.bag filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/episode_rosbags/episode_2_2025-01-21-20-57-56.bag filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/episode_rosbags/episode_2_2025-01-21-21-37-44.bag filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/episode_rosbags/episode_0_2025-01-21-21-36-09.bag filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/episode_rosbags/episode_1_2025-01-21-21-36-35.bag filter=lfs diff=lfs merge=lfs -text +5z33ibf5/episode_rosbags/episode_0_2025-01-21-21-19-25.bag filter=lfs diff=lfs merge=lfs -text +m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/episode_rosbags/episode_1_2025-01-21-20-57-22.bag filter=lfs diff=lfs merge=lfs -text +m9j39xya/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5z33ibf5/episode_rosbags/episode_1_2025-01-21-21-20-18.bag filter=lfs diff=lfs merge=lfs -text +5z33ibf5/episode_rosbags/episode_2_2025-01-21-21-21-10.bag filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/episode_rosbags/episode_1_2025-01-21-21-42-00.bag filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/episode_rosbags/episode_2_2025-01-21-21-42-34.bag filter=lfs diff=lfs merge=lfs -text +8wyk5kwf/episode_rosbags/episode_0_2025-01-21-21-41-29.bag filter=lfs diff=lfs merge=lfs -text +aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/231uxqco/episode_rosbags/episode_0_2025-01-21-21-03-19.bag b/231uxqco/episode_rosbags/episode_0_2025-01-21-21-03-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..7a10468acdf5a7196e18e640d888b40103dc5d96 --- /dev/null +++ b/231uxqco/episode_rosbags/episode_0_2025-01-21-21-03-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceaa7bae4e510a370957f97e51d52d23dba82dcc4e938029e43028491f0c4f33 +size 1074980209 diff --git a/231uxqco/episode_rosbags/episode_1_2025-01-21-21-03-49.bag b/231uxqco/episode_rosbags/episode_1_2025-01-21-21-03-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..a3e64e8b2f0d026feab46a5f4301dd4efa6a01d4 --- /dev/null +++ b/231uxqco/episode_rosbags/episode_1_2025-01-21-21-03-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6879a170e36aaec26f6d5a934850216baad9f4f17f403e329f5929ad10d8657 +size 1075145190 diff --git a/231uxqco/episode_rosbags/episode_4_2025-01-21-21-05-35.bag b/231uxqco/episode_rosbags/episode_4_2025-01-21-21-05-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..d7b6e4b0bb716caa30751c4ab3b8757bdd1ae904 --- /dev/null +++ b/231uxqco/episode_rosbags/episode_4_2025-01-21-21-05-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda2bb1236aa33d292ed014a7f89032d352c9218023290b2db5b2b5309640de6 +size 1320884069 diff --git a/37t5ymwr/episode_rosbags/episode_1_2025-01-21-20-57-22.bag b/37t5ymwr/episode_rosbags/episode_1_2025-01-21-20-57-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..9635e9f82085f5faf4c3cde486ece9911936bf14 --- /dev/null +++ b/37t5ymwr/episode_rosbags/episode_1_2025-01-21-20-57-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3e851ab9568c1faaee98ac3e572c1185f121962ae8a191b5677f95f2a6b450f +size 1498588579 diff --git a/37t5ymwr/episode_rosbags/episode_2_2025-01-21-20-57-56.bag b/37t5ymwr/episode_rosbags/episode_2_2025-01-21-20-57-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..85ea269175fb3a2ceafc5cb98e6ed075c31491c7 --- /dev/null +++ b/37t5ymwr/episode_rosbags/episode_2_2025-01-21-20-57-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6e096a0f0348b24fd8a28e803456e06b755b753eb28acdc9d9ad9547339061 +size 1321418008 diff --git a/37t5ymwr/episode_rosbags/episode_4_2025-01-21-20-59-22.bag b/37t5ymwr/episode_rosbags/episode_4_2025-01-21-20-59-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..6168d7a18aba257596ea3a6803d9f9f8ead6cf89 --- /dev/null +++ b/37t5ymwr/episode_rosbags/episode_4_2025-01-21-20-59-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24b76439f5be2e94737ad303f4e77e24b58a3fdd641a7f52824aa1f1dca0d1d +size 1103146146 diff --git a/5z33ibf5/.hydra/config.yaml b/5z33ibf5/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/5z33ibf5/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5z33ibf5/.hydra/hydra.yaml b/5z33ibf5/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a2ebee791203901b133feb6e7fd7e9e2e84bfeb --- /dev/null +++ b/5z33ibf5/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5z33ibf5/.hydra/overrides.yaml b/5z33ibf5/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5z33ibf5/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5z33ibf5/episode_rosbags/episode_0_2025-01-21-21-19-25.bag b/5z33ibf5/episode_rosbags/episode_0_2025-01-21-21-19-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..2ecbe295070416cec8b4d043ff4c086298b22670 --- /dev/null +++ b/5z33ibf5/episode_rosbags/episode_0_2025-01-21-21-19-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5466324f9c84324e3aa04b5c257554199cc63947f8a78ebbd9cb6f4d762a0484 +size 2610649318 diff --git a/5z33ibf5/episode_rosbags/episode_1_2025-01-21-21-20-18.bag b/5z33ibf5/episode_rosbags/episode_1_2025-01-21-21-20-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..e3514030bade5c8c3886773ae5742d69a5f7719e --- /dev/null +++ b/5z33ibf5/episode_rosbags/episode_1_2025-01-21-21-20-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e509a51ce97f1bd48da5822a1d50bc5d405f578ed3aab53656d4b7999aa1fdae +size 2592040189 diff --git a/5z33ibf5/episode_rosbags/episode_2_2025-01-21-21-21-10.bag b/5z33ibf5/episode_rosbags/episode_2_2025-01-21-21-21-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..f2860ae319f03b3e56c69a4ab27172ea96c5a4c6 --- /dev/null +++ b/5z33ibf5/episode_rosbags/episode_2_2025-01-21-21-21-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e471403dfe5cc63032ec153c7f2af196b9bc491b06682615ffedb95f7913307c +size 2601756742 diff --git a/5z33ibf5/eval_video/0_eval.mp4 b/5z33ibf5/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e88f334376f0aea4fb1a75a543e97e2faf42e386 --- /dev/null +++ b/5z33ibf5/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52952b9d92244b702210bb0b7ecf54272d94ebcdafdb932394ed66139de018ad +size 1496646 diff --git a/5z33ibf5/eval_video/1_eval.mp4 b/5z33ibf5/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..771abfb87ba39b241a52314002603eb54e1091c9 --- /dev/null +++ b/5z33ibf5/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36a449d922a9d3cb1686a75906fc79cfb5289c444b90f06ae867fbdf206f17f +size 1497131 diff --git a/5z33ibf5/eval_video/2_eval.mp4 b/5z33ibf5/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9aa1b11acacdf83625db140dfde0f29792ef0c5 --- /dev/null +++ b/5z33ibf5/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bfe16d417bf0eae1e35767e3953d00274fa7285bf1c4390332b92d49b5de96f +size 1524925 diff --git a/5z33ibf5/eval_video/3_eval.mp4 b/5z33ibf5/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bda0c341fb3c5ea580aeb3ececb876f7c973857e --- /dev/null +++ b/5z33ibf5/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2f247956aef8b976274554f80d2c1d1afc91e008ebaec6d3ca108010709c75c +size 1582269 diff --git a/5z33ibf5/eval_video/4_eval.mp4 b/5z33ibf5/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58f012795ea986376d20d1b6b7950e21dbdd0ef6 --- /dev/null +++ b/5z33ibf5/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b307d108c1b40ec3a037c54c4c4941b5b15cd441d223e640b227116c91f34cd +size 1601611 diff --git a/5z33ibf5/tb/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 b/5z33ibf5/tb/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 new file mode 100644 index 0000000000000000000000000000000000000000..34d0c8832aa5049315d2fc72a9a39333a0e17f74 --- /dev/null +++ b/5z33ibf5/tb/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:186f8b7546d88f4eed9d0fa211b2fa2ea6c0cc8a66c15d1ccaf1c43cea703076 +size 6539 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f299bab9cb35464eed4e0e39a0dc8cde1103b8b7 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737512322 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d95f74d8ed41167431043002578a2fdc814257f6 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[7, 20, 0.35, 0.13707314438925872, 0.6159580349690265, 0.17542183405692, 0.5902580916701045, 0.19973829394623704, 0.5288278013410134, 0.21333447849523368, 0.5130670443151089]]} \ No newline at end of file diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e88f334376f0aea4fb1a75a543e97e2faf42e386 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52952b9d92244b702210bb0b7ecf54272d94ebcdafdb932394ed66139de018ad +size 1496646 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..771abfb87ba39b241a52314002603eb54e1091c9 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36a449d922a9d3cb1686a75906fc79cfb5289c444b90f06ae867fbdf206f17f +size 1497131 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9aa1b11acacdf83625db140dfde0f29792ef0c5 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bfe16d417bf0eae1e35767e3953d00274fa7285bf1c4390332b92d49b5de96f +size 1524925 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bda0c341fb3c5ea580aeb3ececb876f7c973857e --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2f247956aef8b976274554f80d2c1d1afc91e008ebaec6d3ca108010709c75c +size 1582269 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58f012795ea986376d20d1b6b7950e21dbdd0ef6 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b307d108c1b40ec3a037c54c4c4941b5b15cd441d223e640b227116c91f34cd +size 1601611 diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cd9417db3b953684d0358f1f03dce05cae646594 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737512344.066829]: resetting environment +[INFO] [1737512344.071555]: cleared current plan +[INFO] [1737512344.072359]: moving to home +[INFO] [1737512345.074732]: reached home +[INFO] [1737512345.076250]: reset action history +[INFO] [1737512347.791137]: environment reset +Starting episode 0 +[INFO] [1737512347.795354]: resetting environment +[INFO] [1737512347.799639]: cleared current plan +[INFO] [1737512347.800419]: moving to home +[INFO] [1737512348.802850]: reached home +[INFO] [1737512348.804140]: reset action history +[INFO] [1737512351.514226]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737512361.602565]: resetting environment +[INFO] [1737512361.608536]: cleared current plan +[INFO] [1737512361.609563]: moving to home +[INFO] [1737512362.611631]: reached home +[INFO] [1737512362.612975]: reset action history +[INFO] [1737512365.321576]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/episode_rosbags/episode_0_2025-01-21-21-19-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2648747.log +started roslaunch server http://158.130.50.37:45519/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2649062] +process[print_text-2]: started with pid [2649086] +process[rosbag_record-3]: started with pid [2649087] +started bagging! +For topic gripper_width: timestamp difference is 101189745 for nearest: 1737512372579162491 - target: 1737512372477972746 at idx 96 +[INFO] [1737512400.271397]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1737512401.070160]: Stored episode 1. +[INFO] [1737512401.070449]: Saving video... +[INFO] [1737512401.271325]: Video saved! +Starting episode 1 +[INFO] [1737512401.284343]: resetting environment +[INFO] [1737512401.297850]: cleared current plan +[INFO] [1737512401.298455]: moving to home +[INFO] [1737512404.903185]: reached home +[INFO] [1737512404.904354]: reset action history +[INFO] [1737512407.615764]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512414.630790]: resetting environment +[INFO] [1737512414.637108]: cleared current plan +[INFO] [1737512414.637758]: moving to home +[INFO] [1737512415.639205]: reached home +[INFO] [1737512415.639622]: reset action history +[INFO] [1737512418.347720]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/episode_rosbags/episode_1_2025-01-21-21-20-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2648747.log +started roslaunch server http://158.130.50.37:40889/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2649210] +process[print_text-5]: started with pid [2649211] +process[rosbag_record-6]: started with pid [2649235] +started bagging! +For topic gripper_width: timestamp difference is 112526717 for nearest: 1737512419412497105 - target: 1737512419299970388 at idx 15 +For topic gripper_width: timestamp difference is 100327451 for nearest: 1737512445479163368 - target: 1737512445378835917 at idx 249 +For topic gripper_width: timestamp difference is 101364895 for nearest: 1737512446779119536 - target: 1737512446677754641 at idx 249 +For topic gripper_width: timestamp difference is 105899694 for nearest: 1737512452279161575 - target: 1737512452173261881 at idx 249 +[INFO] [1737512453.038391]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737512453.838337]: Stored episode 2. +[INFO] [1737512453.838769]: Saving video... +[INFO] [1737512454.392757]: Video saved! +Starting episode 2 +[INFO] [1737512454.431414]: resetting environment +[INFO] [1737512454.453223]: cleared current plan +[INFO] [1737512454.454833]: moving to home +[WARN] [1737512454.380692]: Plan exhausted +[WARN] [1737512454.420855]: Plan exhausted +[INFO] [1737512458.061684]: reached home +[INFO] [1737512458.065943]: reset action history +[INFO] [1737512460.789738]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512467.102199]: resetting environment +[INFO] [1737512467.110906]: cleared current plan +[INFO] [1737512467.111114]: moving to home +[INFO] [1737512468.112596]: reached home +[INFO] [1737512468.113806]: reset action history +[INFO] [1737512470.823428]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/episode_rosbags/episode_2_2025-01-21-21-21-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2648747.log +started roslaunch server http://158.130.50.37:43591/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2649365] +process[print_text-8]: started with pid [2649366] +process[rosbag_record-9]: started with pid [2649390] +started bagging! +For topic gripper_width: timestamp difference is -105112122 for nearest: 1737512478812493040 - target: 1737512478917605162 at idx 104 +For topic gripper_width: timestamp difference is 102645308 for nearest: 1737512483679187447 - target: 1737512483576542139 at idx 166 +For topic gripper_width: timestamp difference is 102466929 for nearest: 1737512489412494051 - target: 1737512489310027122 at idx 237 +[INFO] [1737512505.752525]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737512506.581306]: Stored episode 3. +[INFO] [1737512506.587126]: Saving video... +[WARN] [1737512507.100658]: Plan exhausted +[WARN] [1737512507.140645]: Plan exhausted +[INFO] [1737512507.070721]: Video saved! +Starting episode 3 +[INFO] [1737512507.120391]: resetting environment +[INFO] [1737512507.145662]: cleared current plan +[INFO] [1737512507.145931]: moving to home +[INFO] [1737512510.752356]: reached home +[INFO] [1737512510.758472]: reset action history +[INFO] [1737512513.549517]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512533.129828]: resetting environment +[INFO] [1737512533.144915]: cleared current plan +[INFO] [1737512533.147726]: moving to home +[INFO] [1737512534.149325]: reached home +[INFO] [1737512534.159630]: reset action history +[INFO] [1737512536.897161]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/episode_rosbags/episode_3_2025-01-21-21-22-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2648747.log +started roslaunch server http://158.130.50.37:40733/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2649541] +process[print_text-11]: started with pid [2649542] +process[rosbag_record-12]: started with pid [2649543] +started bagging! +For topic gripper_width: timestamp difference is 113910194 for nearest: 1737512544879159208 - target: 1737512544765249014 at idx 100 +For topic gripper_width: timestamp difference is 115392148 for nearest: 1737512546679185092 - target: 1737512546563792944 at idx 128 +[INFO] [1737512572.937681]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1737512573.861141]: Stored episode 4. +[INFO] [1737512573.868678]: Saving video... +[INFO] [1737512574.377781]: Video saved! +Starting episode 4 +[INFO] [1737512574.469411]: resetting environment +[INFO] [1737512574.506689]: cleared current plan +[INFO] [1737512574.506871]: moving to home +[WARN] [1737512574.300869]: Plan exhausted +[WARN] [1737512574.341642]: Plan exhausted +[WARN] [1737512574.387984]: Plan exhausted +[WARN] [1737512574.426885]: Plan exhausted +[WARN] [1737512574.461248]: Plan exhausted +[WARN] [1737512574.500782]: Plan exhausted +[INFO] [1737512578.114544]: reached home +[INFO] [1737512578.116574]: reset action history +[INFO] [1737512580.900574]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512588.253288]: resetting environment +[INFO] [1737512588.295903]: cleared current plan +[INFO] [1737512588.298103]: moving to home +[INFO] [1737512589.304532]: reached home +[INFO] [1737512589.313428]: reset action history +[INFO] [1737512592.152229]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/episode_rosbags/episode_4_2025-01-21-21-23-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2648747.log +started roslaunch server http://158.130.50.37:41637/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2649694] +process[print_text-14]: started with pid [2649695] +process[rosbag_record-15]: started with pid [2649696] +started bagging! +For topic gripper_width: timestamp difference is 110738336 for nearest: 1737512623479111730 - target: 1737512623368373394 at idx 249 +[INFO] [1737512628.895769]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737512629.627210]: Stored episode 5. +[INFO] [1737512629.639351]: Saving video... +[INFO] [1737512630.211458]: Video saved! +[INFO] [1737512630.257196]: resetting environment +[INFO] [1737512630.290530]: cleared current plan +[INFO] [1737512630.291321]: moving to home +[WARN] [1737512630.264509]: Plan exhausted +[INFO] [1737512633.895220]: reached home +[INFO] [1737512633.908928]: reset action history +[INFO] [1737512636.694786]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6498c7735acbeb4664d5376693e76111fc36f6ea --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:18:42.776324", + "startedAt": "2025-01-22T02:18:42.329441", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.98625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.942, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 357.1996383666992 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f69145481472d34a4539185f7f2d0264278e0729 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "52952b9d92244b702210bb0b7ecf54272d94ebcdafdb932394ed66139de018ad", "size": 1496646, "path": "media/videos/eval/0_eval_0_52952b9d92244b702210.mp4"}, "global_step": 665, "_timestamp": 1737512641.3707454, "_runtime": 319.0309205055237, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f36a449d922a9d3cb1686a75906fc79cfb5289c444b90f06ae867fbdf206f17f", "size": 1497131, "path": "media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "1bfe16d417bf0eae1e35767e3953d00274fa7285bf1c4390332b92d49b5de96f", "size": 1524925, "path": "media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d2f247956aef8b976274554f80d2c1d1afc91e008ebaec6d3ca108010709c75c", "size": 1582269, "path": "media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "0b307d108c1b40ec3a037c54c4c4941b5b15cd441d223e640b227116c91f34cd", "size": 1601611, "path": "media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://l5071udynioifpufowga68d3m0t8z8xi2tejarl8k1qrqoinp6ruxu5gy0ve1v5u058bhn56f1slutyn7t4flsurppoug4b3tkkfkowot6my3g53drn4dnyz0mq2nyhf/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://dueqyuoirjeskldhrxavjem5dq6q4uj6jce9j08b99gsv6cdg6k3hwpbk616pja151o5j6b24ytfc4p1fyt2qz3jn9qwpalyad6gzu881bvhffsythv6v0k2iwtdjyit:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 7, "total_num_episodes": 20, "total_success_rate": 0.35, "total_success_rate_lb_0.01": 0.13707314438925872, "total_success_rate_ub_0.01": 0.6159580349690265, "total_success_rate_lb_0.025": 0.17542183405692, "total_success_rate_ub_0.025": 0.5902580916701045, "total_success_rate_lb_0.05": 0.19973829394623704, "total_success_rate_ub_0.05": 0.5288278013410134, "total_success_rate_lb_0.1": 0.21333447849523368, "total_success_rate_ub_0.1": 0.5130670443151089, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "7f69fec0f5489086ad7f5bfc7da2cc7aee1b80e239a6fd4a88aa7f44b12d4166", "size": 506, "artifact_path": "wandb-client-artifact://wdarixyqdsfyttggpu0hltznaaebn0uo8htu4kvvm8iqoc7fk99y4vtbfmg3uibkxui15qz9i3oo3sy9fjrt0lkukt8xafqjm6vglv6da8g9c56yq1n9svap6iuzkwi3/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://g4bzvv5a71k739cta8qwzq7eydzk40e9g2ogb0p5ylei3xd8tiexh1dtqvcpi31wm4dkntxd43zyrscy5v0hbk6ppbh56ivkax76ygf3nml9zb7b42znl8o37wfoxwsl:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 339}, "eval/total_success_rate": 0.3499999940395355, "eval/total_success_rate_lb_0.01": 0.13707314431667328, "eval/total_success_rate_ub_0.01": 0.615958034992218, "eval/total_success_rate_lb_0.025": 0.1754218339920044, "eval/total_success_rate_ub_0.025": 0.5902580618858337, "eval/total_success_rate_lb_0.05": 0.1997382938861847, "eval/total_success_rate_ub_0.05": 0.5288277864456177, "eval/total_success_rate_lb_0.1": 0.21333448588848114, "eval/total_success_rate_ub_0.1": 0.5130670666694641} \ No newline at end of file diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug-internal.log b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b1d79f9d3b94d54a45d31496dd5e6b814489bc96 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug-internal.log @@ -0,0 +1,879 @@ +2025-01-21 21:18:42,339 INFO StreamThr :2648942 [internal.py:wandb_internal():85] W&B internal server running at pid: 2648942, started at: 2025-01-21 21:18:42.339227 +2025-01-21 21:18:42,340 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:18:42,352 INFO WriterThread:2648942 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb +2025-01-21 21:18:42,352 DEBUG SenderThread:2648942 [sender.py:send():379] send: header +2025-01-21 21:18:42,359 DEBUG SenderThread:2648942 [sender.py:send():379] send: run +2025-01-21 21:18:42,657 INFO SenderThread:2648942 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files +2025-01-21 21:18:42,657 INFO SenderThread:2648942 [sender.py:_start_run_threads():1188] run started: 5z33ibf5 with start time 1737512322.339825 +2025-01-21 21:18:42,678 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:18:42,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:18:42,757 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:18:42,762 DEBUG HandlerThread:2648942 [system_info.py:__init__():26] System info init +2025-01-21 21:18:42,762 DEBUG HandlerThread:2648942 [system_info.py:__init__():41] System info init done +2025-01-21 21:18:42,762 INFO HandlerThread:2648942 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:18:42,762 INFO SystemMonitor:2648942 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:18:42,762 INFO HandlerThread:2648942 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:18:42,763 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:18:42,763 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:18:42,764 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:18:42,764 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:18:42,765 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:18:42,776 DEBUG HandlerThread:2648942 [system_info.py:probe():152] Probing system +2025-01-21 21:18:42,777 DEBUG HandlerThread:2648942 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:probe():200] Probing system done +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:18:42.776324', 'startedAt': '2025-01-22T02:18:42.329441', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.98625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.942, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 357.1996383666992}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:18:42,782 INFO HandlerThread:2648942 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:18:42,782 INFO HandlerThread:2648942 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:_save_code():44] Saving code +2025-01-21 21:18:42,784 DEBUG HandlerThread:2648942 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:18:42,784 DEBUG HandlerThread:2648942 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:18:42,846 DEBUG HandlerThread:2648942 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:18:42,846 INFO HandlerThread:2648942 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:18:42,849 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:18:42,849 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:18:42,849 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:18:42,850 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:18:43,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:18:43,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:18:43,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:43,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:43,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:43,117 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/jyqneicx-wandb-metadata.json +2025-01-21 21:18:43,123 INFO wandb-upload_2:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/oow9pu73-diff.patch +2025-01-21 21:18:43,157 INFO HandlerThread:2648942 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/tb" + save: true +} +control { + relay_id: "aea2f715713e4a8594b30a5fe7dc8655" +} +_info { + stream_id: "5z33ibf5" +} + +2025-01-21 21:18:43,157 DEBUG HandlerThread:2648942 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:18:43,169 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/zwj3yun0-code/FISH/eval_robot.py +2025-01-21 21:18:43,234 DEBUG SenderThread:2648942 [sender.py:send():379] send: telemetry +2025-01-21 21:18:43,234 DEBUG SenderThread:2648942 [sender.py:send():379] send: tbrecord +2025-01-21 21:18:43,235 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:18:43,235 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 with policy live +2025-01-21 21:18:43,433 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/ppifows0-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:18:43,658 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH +2025-01-21 21:18:44,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:45,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:45,658 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:18:46,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:47,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:48,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:48,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:49,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:50,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:51,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:52,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:53,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:53,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:54,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:55,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:56,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:57,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:58,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:58,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:58,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:58,230 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:59,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:00,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:01,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:02,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:03,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:04,066 DEBUG SenderThread:2648942 [sender.py:send():379] send: config +2025-01-21 21:19:04,066 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:04,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:05,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:05,663 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:06,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:07,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:07,664 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:08,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:09,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:09,664 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:09,845 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:10,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:11,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:11,665 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:12,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:13,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:13,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:13,666 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:14,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:15,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:15,200 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:15,666 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:19:16,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:17,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:18,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:19,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:20,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:21,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:21,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:22,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:23,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:23,668 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:24,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:25,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:26,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:26,636 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:27,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:27,669 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:28,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:28,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:28,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:29,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:30,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:31,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:32,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:32,260 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:33,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:33,671 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:34,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:35,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:36,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:37,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:38,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:38,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:39,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:40,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:41,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:42,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:42,765 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:19:42,766 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:19:43,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:43,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:43,253 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:44,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:45,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:46,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:47,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:48,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:49,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:49,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:50,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:51,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:52,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:53,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:54,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:54,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:55,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:56,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:57,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:58,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:58,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:58,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:59,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:59,183 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:00,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:01,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:01,313 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:20:01,314 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 with policy now +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:20:01,315 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:20:01,551 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/j2rtqp81-media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:20:01,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos +2025-01-21 21:20:01,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media +2025-01-21 21:20:01,982 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/by9g8eas-media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:02,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:03,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:03,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:04,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:04,903 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:05,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:06,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:07,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:07,681 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:08,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:09,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:09,682 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:10,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:10,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:11,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:12,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:12,766 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:20:13,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:13,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:14,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:14,683 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:20:14,935 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/owyry9cl-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:20:15,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:15,639 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:15,684 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:16,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:17,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:17,684 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:18,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:19,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:19,685 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:20,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:21,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:21,465 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:21,686 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:22,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:23,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:24,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:25,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:26,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:27,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:27,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:28,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:28,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:28,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:29,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:30,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:31,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:32,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:32,253 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:33,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:34,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:35,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:36,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:37,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:38,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:38,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:39,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:40,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:41,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:42,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:42,767 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:20:43,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:43,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:43,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:44,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:45,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:46,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:47,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:47,694 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:48,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:49,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:49,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:50,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:51,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:52,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:53,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:53,696 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:54,135 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:54,384 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:54,429 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:20:54,429 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 with policy now +2025-01-21 21:20:54,430 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:20:54,430 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:20:54,431 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:20:54,432 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:20:54,672 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/tgaverpi-media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:54,696 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:20:54,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:54,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:20:55,002 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/1o8zgjky-media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:55,135 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:55,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:56,138 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:56,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:57,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:58,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:58,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:58,140 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:59,140 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:59,698 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:00,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:00,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:01,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:01,699 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:02,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:03,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:04,142 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:05,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:06,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:06,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:07,142 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:07,674 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:21:07,674 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:21:07,675 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:21:07,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:21:07,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:21:08,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:09,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:09,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:10,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:11,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:11,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:11,702 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:12,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:12,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:21:13,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:13,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:13,703 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:14,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:15,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:16,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:16,254 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:17,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:18,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:19,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:20,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:21,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:21,706 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:22,050 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:22,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:23,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:24,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:25,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:25,707 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:26,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:27,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:27,746 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:28,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:28,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:28,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:29,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:30,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:31,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:31,709 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:32,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:33,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:33,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:34,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:35,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:36,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:37,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:38,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:38,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:39,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:40,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:41,148 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:42,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:42,768 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:21:43,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:43,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:43,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:44,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:45,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:46,150 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:47,115 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:21:47,116 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 with policy now +2025-01-21 21:21:47,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:21:47,119 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:21:47,119 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:21:47,138 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:21:47,150 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:47,354 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/vpnbad59-media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:21:48,007 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/vjkzjv3s-media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:48,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:48,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:49,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:49,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:49,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:50,164 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:51,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:51,716 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:52,158 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:53,158 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:54,163 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:54,597 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:55,159 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:55,717 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:56,160 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:57,163 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:58,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:58,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:58,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:59,164 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:00,170 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:00,198 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:01,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:02,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:03,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:04,167 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:05,166 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:06,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:06,166 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:07,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:08,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:09,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:10,167 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:11,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:12,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:12,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:12,768 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:22:13,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:13,146 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:13,174 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:13,456 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:22:13,456 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:22:13,457 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:22:13,723 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:22:13,724 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:22:13,983 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/khidhdah-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:22:14,193 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:15,181 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:15,724 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:16,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:17,181 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:17,725 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:17,941 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:18,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:19,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:19,725 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:20,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:21,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:22,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:23,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:23,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:24,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:25,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:25,727 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:26,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:27,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:27,728 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:28,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:28,117 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:28,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:28,250 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:29,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:30,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:31,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:32,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:33,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:34,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:34,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:35,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:36,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:37,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:38,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:39,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:40,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:40,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:41,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:42,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:42,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:22:43,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:43,117 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:43,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:44,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:45,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:45,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:46,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:47,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:48,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:49,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:50,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:51,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:51,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:52,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:53,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:53,737 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:54,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:54,467 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:22:54,467 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 with policy now +2025-01-21 21:22:54,470 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:22:54,470 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:22:54,470 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:22:54,486 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:22:54,706 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/bdtb2yzm-media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:54,737 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:22:54,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:54,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:22:55,025 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/x0695abo-media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:55,191 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:55,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:56,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:56,507 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:56,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:57,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:58,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:58,157 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:58,192 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:59,212 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:59,739 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:00,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:01,193 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:01,740 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:01,945 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:02,194 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:03,199 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:04,201 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:05,205 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:06,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:07,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:07,209 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:08,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:08,525 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:23:08,525 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:23:08,526 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:23:08,742 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:23:08,742 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:23:09,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:09,743 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:10,212 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:11,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:11,743 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:12,153 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:12,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:12,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:23:13,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:13,124 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:13,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:13,744 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:14,214 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:15,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:15,745 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:16,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:17,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:17,423 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:18,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:19,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:20,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:21,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:22,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:23,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:23,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:24,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:25,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:26,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:27,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:28,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:28,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:28,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:28,272 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:29,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:30,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:31,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:32,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:33,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:34,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:34,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:35,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:36,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:37,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:38,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:39,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:40,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:40,222 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:41,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:42,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:42,770 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:23:43,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:43,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:43,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:44,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:45,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:45,529 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:45,756 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:46,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:47,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:48,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:49,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:49,757 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:50,231 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:50,251 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:23:50,251 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 with policy now +2025-01-21 21:23:50,297 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:23:50,298 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:23:50,298 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:23:50,311 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:23:50,481 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/e6vqeati-media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:23:51,072 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/ajfhb57u-media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:51,236 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:51,312 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:51,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:52,226 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:52,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:53,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:54,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:55,246 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:55,759 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:56,245 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:56,695 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:57,232 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:57,760 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:58,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:58,127 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:58,238 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:59,243 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:00,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:00,588 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:00,588 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:00,589 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:00,761 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:00,761 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:00,804 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:24:00,804 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:24:01,037 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/rmqtnflz-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:01,065 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:24:01,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:01,260 INFO wandb-upload_0:2648942 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa_4wwwtd +2025-01-21 21:24:01,449 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:24:01,460 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:24:01,676 INFO SenderThread:2648942 [sender.py:send_request_log_artifact():1525] logged artifact run-5z33ibf5-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY2NTU2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NDk1MQ==', 'latestArtifact': None}} +2025-01-21 21:24:01,676 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:24:01,676 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-21 21:24:01,677 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:01,677 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,677 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send():379] send: telemetry +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,678 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,679 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,679 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,681 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,681 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,682 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,682 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,682 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,682 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,683 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,683 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,684 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,684 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,684 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,684 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,685 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,685 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,685 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval +2025-01-21 21:24:01,763 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table +2025-01-21 21:24:01,881 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/84vvl9wq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:01,965 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/2rogxjlf-media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:02,103 INFO wandb-upload_2:2648942 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2svt4rn5 +2025-01-21 21:24:02,138 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/h2m904lt-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:02,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:02,467 INFO SenderThread:2648942 [sender.py:send_request_log_artifact():1525] logged artifact run-5z33ibf5-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY2NTU4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NDk1OQ==', 'latestArtifact': None}} +2025-01-21 21:24:02,467 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:24:02,467 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json with policy now +2025-01-21 21:24:02,468 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:02,468 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:02,468 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:02,469 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:02,712 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/rau7ntk6-media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:02,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:03,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:03,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:04,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:05,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:06,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:07,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:08,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:08,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:09,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:10,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:11,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:12,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:12,770 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:24:13,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:24:13,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:24:13,192 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:13,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:13,766 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:24:14,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:15,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:16,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:17,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:18,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:19,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:19,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:20,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:21,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:21,981 DEBUG SenderThread:2648942 [sender.py:send():379] send: exit +2025-01-21 21:24:21,981 INFO SenderThread:2648942 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:24:21,981 INFO SenderThread:2648942 [sender.py:send_exit():588] handling runtime: 339 +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:send_exit():594] send defer +2025-01-21 21:24:21,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,982 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:24:21,982 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:24:21,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,982 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:24:21,983 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:21,983 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:24:21,983 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:24:21,983 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:24:21,983 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:24:21,983 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:24:21,984 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:24:22,060 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:22,060 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:24:22,061 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:22,061 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:24:22,770 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:23,770 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:28,596 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:28,596 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:28,597 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,598 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:24:28,598 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:28,598 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,599 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:24:28,600 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:28,600 DEBUG SenderThread:2648942 [sender.py:send():379] send: summary +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:28,604 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:24:28,604 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,605 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:24:28,605 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:24:28,605 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:28,605 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,605 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:24:28,605 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:24:28,773 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:28,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:30,583 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:24:30,583 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:30,584 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:30,584 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:24:30,584 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:30,584 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:24:30,584 INFO SenderThread:2648942 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:24:30,587 INFO SenderThread:2648942 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:24:30,589 INFO SenderThread:2648942 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:24:30,598 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:24:30,598 DEBUG SenderThread:2648942 [sender.py:send():379] send: artifact +2025-01-21 21:24:30,601 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:30,601 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:24:30,773 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:24:30,812 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:24:30,983 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml config.yaml +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json wandb-summary.json +2025-01-21 21:24:31,775 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt requirements.txt +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log output.log +2025-01-21 21:24:31,777 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch diff.patch +2025-01-21 21:24:31,782 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:24:31,782 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:24:31,783 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:31,783 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:31,783 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:24:31,784 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:31,784 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:24:31,784 INFO SenderThread:2648942 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:24:31,968 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:24:31,969 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:31,984 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:31,984 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:32,009 INFO wandb-upload_2:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt +2025-01-21 21:24:32,018 INFO wandb-upload_4:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:32,645 INFO wandb-upload_3:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/f6ont0lf-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:32,845 INFO Thread-11 (_thread_body):2648942 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:24:32,845 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:32,845 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:24:32,846 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:32,846 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:24:32,846 INFO SenderThread:2648942 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:24:32,847 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:24:32,847 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:32,847 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:24:32,847 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:32,848 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:24:32,848 INFO SenderThread:2648942 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:24:32,984 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,132 INFO SenderThread:2648942 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:24:33,132 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:24:33,132 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:33,132 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:24:33,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:33,133 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send():379] send: final +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send():379] send: footer +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:24:33,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,134 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:24:33,135 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:24:33,136 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,136 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:24:33,137 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:24:33,137 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:33,206 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,206 INFO MainThread:2648942 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:24:33,207 INFO MainThread:2648942 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:24:33,207 INFO MainThread:2648942 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:24:33,207 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:24:33,207 INFO HandlerThread:2648942 [handler.py:finish():882] shutting down handler +2025-01-21 21:24:34,136 INFO WriterThread:2648942 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [sender.py:finish():1615] shutting down sender +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [file_pusher.py:join():175] waiting for file pusher diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug.log b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6b7096dafa2e4d7df4475e94f7d018845360e01a --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Configure stats pid to 2648747 +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/settings +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug.log +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug-internal.log +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5'} +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():619] starting backend +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():623] setting up manager +2025-01-21 21:18:42,338 INFO MainThread:2648747 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:18:42,339 INFO MainThread:2648747 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:18:42,351 INFO MainThread:2648747 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:18:42,357 INFO MainThread:2648747 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:18:42,678 INFO MainThread:2648747 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:18:42,752 INFO MainThread:2648747 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:18:42,752 INFO MainThread:2648747 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:18:43,114 INFO MainThread:2648747 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:18:43,118 INFO MainThread:2648747 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:18:43,118 INFO MainThread:2648747 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/tb, True +2025-01-21 21:19:04,065 INFO MainThread:2648747 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:24:34,208 WARNING MsgRouterThr:2648747 [router.py:message_loop():77] message_loop has been closed diff --git a/6yr9kinc/.hydra/config.yaml b/6yr9kinc/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/6yr9kinc/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/6yr9kinc/.hydra/hydra.yaml b/6yr9kinc/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0bee778e6d8a41ad33e732ec1b527a41258dc9f --- /dev/null +++ b/6yr9kinc/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/6yr9kinc + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/6yr9kinc/.hydra/overrides.yaml b/6yr9kinc/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/6yr9kinc/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/6yr9kinc/eval_policy.log b/6yr9kinc/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..b6d566bdbf5e061d57d81144b679a42a4dc4ef74 --- /dev/null +++ b/6yr9kinc/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 21:18:33,733][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:18:33,737][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 21:18:36,249][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/8wyk5kwf/.hydra/config.yaml b/8wyk5kwf/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/8wyk5kwf/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8wyk5kwf/.hydra/hydra.yaml b/8wyk5kwf/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c1ed5af3556727c1a4506c77755ccf9732b5fa9 --- /dev/null +++ b/8wyk5kwf/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8wyk5kwf/.hydra/overrides.yaml b/8wyk5kwf/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8wyk5kwf/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8wyk5kwf/episode_rosbags/aligned_depth_to_color_K.npy b/8wyk5kwf/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8wyk5kwf/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8wyk5kwf/episode_rosbags/cam_tf_world.npy b/8wyk5kwf/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/8wyk5kwf/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/8wyk5kwf/episode_rosbags/color_K.npy b/8wyk5kwf/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8wyk5kwf/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8wyk5kwf/episode_rosbags/depth_K.npy b/8wyk5kwf/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/8wyk5kwf/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/8wyk5kwf/episode_rosbags/episode_0_2025-01-21-21-41-29.bag b/8wyk5kwf/episode_rosbags/episode_0_2025-01-21-21-41-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..41f7dbfa3ac2f1a2575b3d5131e1e3a7af40701a --- /dev/null +++ b/8wyk5kwf/episode_rosbags/episode_0_2025-01-21-21-41-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12fc6c7955fda91495ee3aada8f40ef61b3b93b00e6d410a236709f42932cf74 +size 1257913700 diff --git a/8wyk5kwf/episode_rosbags/episode_1_2025-01-21-21-42-00.bag b/8wyk5kwf/episode_rosbags/episode_1_2025-01-21-21-42-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..ceb2a6792217990ea4d91998bd5cf29d296a7694 --- /dev/null +++ b/8wyk5kwf/episode_rosbags/episode_1_2025-01-21-21-42-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf1016ab53f3d144b5c322473b9cd5ab29d6506bf8dab28879a44192b57aa6c +size 1343202659 diff --git a/8wyk5kwf/episode_rosbags/episode_2_2025-01-21-21-42-34.bag b/8wyk5kwf/episode_rosbags/episode_2_2025-01-21-21-42-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..09941e9dc8741671dce851090880adeb7831ee0e --- /dev/null +++ b/8wyk5kwf/episode_rosbags/episode_2_2025-01-21-21-42-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e868df6152615de019ad76ec20754eac5aeab7f6d0f1e701221a789b5b397adf +size 1131697979 diff --git a/8wyk5kwf/eval_robot.log b/8wyk5kwf/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..02856008461bdcaebf02c97982533c1ac89a9009 --- /dev/null +++ b/8wyk5kwf/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:40:59,961][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:40:59,964][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/8wyk5kwf/eval_video/0_eval.mp4 b/8wyk5kwf/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbc72ee5d11e9ce31c14943541d41ea1996e73f9 --- /dev/null +++ b/8wyk5kwf/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42455952124d4dc1494a794ead57978b2c8afd8bbb95b5946273d6a77f3e4734 +size 836964 diff --git a/8wyk5kwf/eval_video/1_eval.mp4 b/8wyk5kwf/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24b1d1ab632e6bc953e7cfd2e47dcfdd5ddafe89 --- /dev/null +++ b/8wyk5kwf/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d49fa2235834d91505b2ed986ef93f7c337e8bd15a86c32de0ac9065021b739 +size 868076 diff --git a/8wyk5kwf/eval_video/2_eval.mp4 b/8wyk5kwf/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d047ccf1cc60ec4a7acc884263a35c39bc03aaad --- /dev/null +++ b/8wyk5kwf/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59a19bbca09918743b4ca42425de42894165a00081fd4e7f080290d9a85ab51 +size 784931 diff --git a/8wyk5kwf/eval_video/3_eval.mp4 b/8wyk5kwf/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e16123901c8c428795210f9b7689e520005c91a --- /dev/null +++ b/8wyk5kwf/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41dd8f6943cd079f1ca457d10191d663e4348f9fde2513ed4b2cd849d37ca46e +size 1510000 diff --git a/8wyk5kwf/eval_video/4_eval.mp4 b/8wyk5kwf/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..195ff033f433721e00e4cb169d71b08eef3c4909 --- /dev/null +++ b/8wyk5kwf/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812bd740546f7b5fae32b425bca424d6dff0d99fc3c07a9e3a3bc7d001cd30b7 +size 937071 diff --git a/8wyk5kwf/tb/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 b/8wyk5kwf/tb/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 new file mode 100644 index 0000000000000000000000000000000000000000..56259609aa72272cb1aa1c0557d91d926bbd7076 --- /dev/null +++ b/8wyk5kwf/tb/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:671cf9c6ad56f7d24e12ef37d92d63c1c7f6925619b7e74bbcfe2726416f4dd1 +size 6519 diff --git a/8wyk5kwf/wandb/debug-internal.log b/8wyk5kwf/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6ca8656e8b3ba0daeb489ea06d6180019c145e9e --- /dev/null +++ b/8wyk5kwf/wandb/debug-internal.log @@ -0,0 +1,694 @@ +2025-01-21 21:41:05,923 INFO StreamThr :2654427 [internal.py:wandb_internal():85] W&B internal server running at pid: 2654427, started at: 2025-01-21 21:41:05.923231 +2025-01-21 21:41:05,925 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:41:05,935 INFO WriterThread:2654427 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb +2025-01-21 21:41:05,935 DEBUG SenderThread:2654427 [sender.py:send():379] send: header +2025-01-21 21:41:05,943 DEBUG SenderThread:2654427 [sender.py:send():379] send: run +2025-01-21 21:41:06,732 INFO SenderThread:2654427 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files +2025-01-21 21:41:06,732 INFO SenderThread:2654427 [sender.py:_start_run_threads():1188] run started: 8wyk5kwf with start time 1737513665.923216 +2025-01-21 21:41:06,751 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:41:06,751 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:41:06,800 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:41:06,805 DEBUG HandlerThread:2654427 [system_info.py:__init__():26] System info init +2025-01-21 21:41:06,805 DEBUG HandlerThread:2654427 [system_info.py:__init__():41] System info init done +2025-01-21 21:41:06,805 INFO HandlerThread:2654427 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:41:06,805 INFO SystemMonitor:2654427 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:41:06,805 INFO HandlerThread:2654427 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:41:06,806 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:41:06,806 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:41:06,807 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:41:06,807 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:41:06,808 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:41:06,818 DEBUG HandlerThread:2654427 [system_info.py:probe():152] Probing system +2025-01-21 21:41:06,820 DEBUG HandlerThread:2654427 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:probe():200] Probing system done +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:41:06.818942', 'startedAt': '2025-01-22T02:41:05.916233', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.31625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.154, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.356, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 398.9217071533203}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:41:06,825 INFO HandlerThread:2654427 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:41:06,825 INFO HandlerThread:2654427 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:_save_code():44] Saving code +2025-01-21 21:41:06,826 DEBUG HandlerThread:2654427 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:41:06,826 DEBUG HandlerThread:2654427 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:41:06,888 DEBUG HandlerThread:2654427 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:41:06,889 INFO HandlerThread:2654427 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:41:06,892 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:41:07,123 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/oziqii6d-wandb-metadata.json +2025-01-21 21:41:07,133 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:41:07,133 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:41:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:07,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:07,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:07,173 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/aajryftu-diff.patch +2025-01-21 21:41:07,177 INFO HandlerThread:2654427 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/tb" + save: true +} +control { + relay_id: "20885cd1050a4c509835d88f39296cfe" +} +_info { + stream_id: "8wyk5kwf" +} + +2025-01-21 21:41:07,177 DEBUG HandlerThread:2654427 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:41:07,179 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/gvff86vm-code/FISH/eval_robot.py +2025-01-21 21:41:07,287 DEBUG SenderThread:2654427 [sender.py:send():379] send: telemetry +2025-01-21 21:41:07,287 DEBUG SenderThread:2654427 [sender.py:send():379] send: tbrecord +2025-01-21 21:41:07,288 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:07,288 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 with policy live +2025-01-21 21:41:07,503 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/9mxep34g-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:07,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:07,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH +2025-01-21 21:41:08,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:09,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:09,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:10,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:11,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:11,288 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:12,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:13,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:14,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:15,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:16,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:16,880 DEBUG SenderThread:2654427 [sender.py:send():379] send: config +2025-01-21 21:41:16,880 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:17,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:18,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:19,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:19,735 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:20,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:21,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:21,736 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:22,133 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:22,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:22,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:22,227 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:23,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:23,736 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:24,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:25,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:25,737 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:26,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:27,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:27,738 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:28,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:28,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:29,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:29,738 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:30,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:31,137 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:31,739 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:32,138 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:33,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:33,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:34,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:35,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:36,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:37,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:37,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:38,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:38,199 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:38,741 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:41:39,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:40,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:41,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:42,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:43,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:44,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:44,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:45,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:46,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:47,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:47,677 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:47,678 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 with policy now +2025-01-21 21:41:47,678 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:41:47,679 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:41:47,679 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:41:47,681 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos +2025-01-21 21:41:47,900 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/j8bg8nph-media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:48,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:48,175 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/a9mud29r-media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:49,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:49,697 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:49,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:50,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:51,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:52,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:52,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:52,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:53,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:53,745 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:54,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:54,953 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:55,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:55,746 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:56,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:57,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:57,747 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:58,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:59,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:59,747 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:00,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:00,269 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:01,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:02,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:03,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:03,749 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:04,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:05,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:06,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:06,258 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:06,808 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:42:06,809 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:42:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:07,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:07,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:08,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:09,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:10,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:11,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:11,751 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:12,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:12,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:13,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:14,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:15,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:16,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:17,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:17,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:18,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:19,155 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:20,158 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:20,213 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:42:20,214 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 with policy now +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:20,215 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:20,444 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/3lrwmdyo-media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:42:21,042 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/ehwzovwb-media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:21,157 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:21,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:22,137 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:22,138 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:22,185 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:22,288 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:23,157 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:24,159 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:25,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:25,756 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:26,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:27,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:27,756 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:28,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:28,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:29,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:30,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:30,551 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:30,551 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:30,552 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:30,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:30,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:42:31,018 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/1urm2fut-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:42:31,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:31,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:32,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:33,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:33,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:33,759 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:34,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:35,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:35,759 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:36,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:36,809 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:42:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:37,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:37,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:38,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:38,286 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:39,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:40,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:41,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:42,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:43,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:44,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:44,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:45,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:46,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:47,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:47,763 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:48,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:49,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:49,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:50,167 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:50,481 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:42:50,481 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:42:50,482 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 with policy now +2025-01-21 21:42:50,482 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:50,482 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:50,482 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:50,710 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/vfahnxli-media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:50,764 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:50,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:50,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:42:51,048 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/zq6tkm2w-media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:51,170 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:51,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:52,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:52,141 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:52,185 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:53,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:54,170 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:55,045 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:55,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:55,766 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:56,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:57,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:57,767 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:58,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:59,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:00,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:00,171 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:00,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:01,179 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:01,371 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:01,372 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:01,372 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:01,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:01,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:02,179 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:03,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:03,769 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:04,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:05,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:05,684 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:05,769 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:06,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:06,810 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:43:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:07,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:07,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:08,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:09,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:10,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:11,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:11,302 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:12,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:13,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:14,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:15,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:16,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:17,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:17,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:18,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:19,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:20,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:21,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:22,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:22,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:22,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:22,292 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:23,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:24,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:25,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:26,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:27,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:27,777 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:28,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:28,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:29,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:30,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:31,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:32,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:33,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:33,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:34,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:35,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:36,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:36,810 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:43:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:37,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:37,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:38,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:38,202 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:39,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:40,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:40,549 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:43:40,549 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:43:40,550 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 with policy now +2025-01-21 21:43:40,550 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:40,550 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:40,576 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:43:40,800 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/h5tfg0fj-media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:41,100 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/3f5ebn25-media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:41,183 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:41,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:42,182 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:43,184 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:43,922 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:44,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:45,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:45,784 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:46,189 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:47,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:47,784 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:48,195 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:49,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:49,191 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:49,785 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:50,052 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/917ru0t5-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:50,190 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:50,455 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:50,455 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:50,455 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:50,785 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:51,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:51,786 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:52,138 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:52,138 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:52,199 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:53,196 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:54,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:54,531 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:55,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:55,787 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:56,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:57,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:58,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:59,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:00,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:00,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:01,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:02,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:03,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:04,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:05,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:06,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:06,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:06,811 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:07,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:07,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:07,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:08,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:09,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:10,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:11,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:11,299 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:12,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:13,200 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:13,793 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:14,072 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:14,072 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 with policy now +2025-01-21 21:44:14,077 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:14,077 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:14,077 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:14,078 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:14,197 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:14,310 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/u749tsej-media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:44:15,091 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/5qjme4xh-media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:15,201 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:15,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:16,202 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:16,795 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:17,148 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:17,204 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:18,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:19,206 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:19,796 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:20,223 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:21,206 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:21,796 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:22,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:22,144 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:22,216 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:22,299 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:23,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:23,797 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:24,099 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:44:24,099 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:44:24,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:24,298 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:24,619 INFO wandb-upload_1:2654427 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi6fnlpzl +2025-01-21 21:44:24,678 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:44:24,688 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table +2025-01-21 21:44:25,052 INFO SenderThread:2654427 [sender.py:send_request_log_artifact():1525] logged artifact run-8wyk5kwf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4OTY0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5OTI1OQ==', 'latestArtifact': None}} +2025-01-21 21:44:25,052 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:25,052 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-21 21:44:25,052 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:25,053 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,066 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,066 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:25,066 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,084 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send():379] send: telemetry +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,085 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,107 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,107 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,127 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,127 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,128 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,128 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,151 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,151 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,173 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,173 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,195 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,195 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,196 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,196 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,209 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:25,217 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,217 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,236 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,236 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,242 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/mg9l7n33-media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:25,260 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,260 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:44:25,288 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/ztuohfpb-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:25,551 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/jcmpzxi2-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:25,700 INFO wandb-upload_0:2654427 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzqo7q131 +2025-01-21 21:44:25,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:25,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:26,210 INFO SenderThread:2654427 [sender.py:send_request_log_artifact():1525] logged artifact run-8wyk5kwf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4OTY3NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5OTI2NA==', 'latestArtifact': None}} +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:26,210 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json with policy now +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:26,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:26,232 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:26,491 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/t3s3dhil-media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:26,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:27,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:27,799 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:28,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:28,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:29,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:30,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:31,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:32,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:33,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:33,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:34,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:35,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:36,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:36,811 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:37,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:37,136 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:37,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:38,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:38,300 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:39,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:40,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:41,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:41,553 DEBUG SenderThread:2654427 [sender.py:send():379] send: exit +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():588] handling runtime: 214 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():594] send defer +2025-01-21 21:44:41,554 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,554 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:44:41,554 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:44:41,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:44:41,555 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,555 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:44:41,555 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:44:41,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:44:41,555 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:44:41,555 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:44:41,556 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:44:41,556 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:44:41,610 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,610 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:44:41,610 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:44:41,610 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:41,610 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,610 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:44:41,611 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,611 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:44:41,611 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:44:41,611 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,611 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:44:41,805 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:43,806 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:48,502 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:48,502 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:44:48,502 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:44:48,503 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:48,503 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,503 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:44:48,504 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:48,504 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:48,505 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:48,506 DEBUG SenderThread:2654427 [sender.py:send():379] send: summary +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:48,506 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:44:48,506 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,506 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:44:48,507 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,507 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:44:48,510 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:48,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:48,656 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:44:48,656 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:48,656 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,656 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:44:48,656 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,656 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:44:48,808 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:48,808 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:44:49,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:50,006 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:44:50,006 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:50,007 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,007 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:44:50,007 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,007 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:44:50,007 INFO SenderThread:2654427 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:44:50,009 INFO SenderThread:2654427 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:44:50,011 INFO SenderThread:2654427 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:44:50,021 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:44:50,021 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,021 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:44:50,021 DEBUG SenderThread:2654427 [sender.py:send():379] send: artifact +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:44:50,236 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:44:50,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml config.yaml +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json wandb-summary.json +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt requirements.txt +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log output.log +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch diff.patch +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:44:50,814 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:50,817 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,817 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:44:50,817 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,817 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:44:50,817 INFO SenderThread:2654427 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:44:51,009 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:51,037 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:44:51,062 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/w8jxwy3h-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:51,072 INFO wandb-upload_3:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt +2025-01-21 21:44:51,083 INFO wandb-upload_4:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:51,283 INFO Thread-11 (_thread_body):2654427 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:44:51,283 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,283 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:44:51,283 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,283 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:44:51,284 INFO SenderThread:2654427 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:44:51,285 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,285 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:44:51,285 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:44:51,490 INFO SenderThread:2654427 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:44:51,490 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:44:51,490 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,491 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,491 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:44:51,491 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:44:51,491 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,491 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send():379] send: final +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send():379] send: footer +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,492 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:51,492 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:51,493 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:44:51,494 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:44:51,495 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:44:51,495 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:51,544 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:51,544 INFO MainThread:2654427 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:44:51,545 INFO MainThread:2654427 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:44:51,545 INFO MainThread:2654427 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:44:51,545 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:44:51,545 INFO HandlerThread:2654427 [handler.py:finish():882] shutting down handler +2025-01-21 21:44:52,493 INFO WriterThread:2654427 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [sender.py:finish():1615] shutting down sender +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [file_pusher.py:join():175] waiting for file pusher diff --git a/8wyk5kwf/wandb/debug.log b/8wyk5kwf/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7b91939e67b993ceb3e08c7f0820951807923eb2 --- /dev/null +++ b/8wyk5kwf/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Configure stats pid to 2654253 +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/settings +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug.log +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug-internal.log +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf'} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():619] starting backend +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():623] setting up manager +2025-01-21 21:41:05,922 INFO MainThread:2654253 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:41:05,923 INFO MainThread:2654253 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:41:05,934 INFO MainThread:2654253 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:41:05,941 INFO MainThread:2654253 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:41:06,750 INFO MainThread:2654253 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:41:06,795 INFO MainThread:2654253 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:41:06,795 INFO MainThread:2654253 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:41:07,134 INFO MainThread:2654253 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:41:07,137 INFO MainThread:2654253 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:41:07,137 INFO MainThread:2654253 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/tb, True +2025-01-21 21:41:16,879 INFO MainThread:2654253 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:44:52,546 WARNING MsgRouterThr:2654253 [router.py:message_loop():77] message_loop has been closed diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66cfba8d16531bd25bb9155bedff2995caedac72 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737513665 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9a8e07dc56e73f3d466ee7d014427d469cb81e82 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[8, 30, 0.26666666666666666, 0.10985761888677481, 0.4807906448879513, 0.13731220371824376, 0.45738425851720543, 0.15524378425817548, 0.4084890708510316, 0.1665180363427985, 0.39319007100810954]]} \ No newline at end of file diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbc72ee5d11e9ce31c14943541d41ea1996e73f9 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42455952124d4dc1494a794ead57978b2c8afd8bbb95b5946273d6a77f3e4734 +size 836964 diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24b1d1ab632e6bc953e7cfd2e47dcfdd5ddafe89 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d49fa2235834d91505b2ed986ef93f7c337e8bd15a86c32de0ac9065021b739 +size 868076 diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d047ccf1cc60ec4a7acc884263a35c39bc03aaad --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59a19bbca09918743b4ca42425de42894165a00081fd4e7f080290d9a85ab51 +size 784931 diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e16123901c8c428795210f9b7689e520005c91a --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41dd8f6943cd079f1ca457d10191d663e4348f9fde2513ed4b2cd849d37ca46e +size 1510000 diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..195ff033f433721e00e4cb169d71b08eef3c4909 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812bd740546f7b5fae32b425bca424d6dff0d99fc3c07a9e3a3bc7d001cd30b7 +size 937071 diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d6fc229088da3ebc102a2352b8879535b1f6678c --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737513676.880080]: resetting environment +[INFO] [1737513676.891354]: cleared current plan +[INFO] [1737513676.892291]: moving to home +[INFO] [1737513677.894696]: reached home +[INFO] [1737513677.896007]: reset action history +[INFO] [1737513680.604921]: environment reset +Starting episode 0 +[INFO] [1737513680.609189]: resetting environment +[INFO] [1737513680.613285]: cleared current plan +[INFO] [1737513680.614176]: moving to home +[INFO] [1737513681.616601]: reached home +[INFO] [1737513681.618401]: reset action history +[INFO] [1737513684.329117]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737513685.903430]: resetting environment +[INFO] [1737513685.908181]: cleared current plan +[INFO] [1737513685.909138]: moving to home +[INFO] [1737513686.911329]: reached home +[INFO] [1737513686.912502]: reset action history +[INFO] [1737513689.622116]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/episode_rosbags/episode_0_2025-01-21-21-41-29.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2654253.log +started roslaunch server http://158.130.50.37:35457/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2654533] +process[print_text-2]: started with pid [2654534] +process[rosbag_record-3]: started with pid [2654558] +started bagging! +preempting episode +[INFO] [1737513706.709905]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737513707.427844]: Stored episode 1. +[INFO] [1737513707.428094]: Saving video... +[INFO] [1737513707.664522]: Video saved! +Starting episode 1 +[INFO] [1737513707.681920]: resetting environment +[INFO] [1737513707.693669]: cleared current plan +[INFO] [1737513707.694016]: moving to home +[INFO] [1737513711.199805]: reached home +[INFO] [1737513711.200794]: reset action history +[INFO] [1737513713.912059]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513717.219697]: resetting environment +[INFO] [1737513717.223464]: cleared current plan +[INFO] [1737513717.223636]: moving to home +[INFO] [1737513718.225061]: reached home +[INFO] [1737513718.225473]: reset action history +[INFO] [1737513720.933496]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/episode_rosbags/episode_1_2025-01-21-21-42-00.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2654253.log +started roslaunch server http://158.130.50.37:45845/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2654679] +process[print_text-5]: started with pid [2654680] +process[rosbag_record-6]: started with pid [2654681] +started bagging! +For topic gripper_width: timestamp difference is -210883170 for nearest: 1737513721912445039 - target: 1737513722123328209 at idx 17 +For topic gripper_width: timestamp difference is 105591270 for nearest: 1737513730179105255 - target: 1737513730073513985 at idx 107 +preempting episode +[INFO] [1737513739.114983]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737513739.836855]: Stored episode 2. +[INFO] [1737513739.837122]: Saving video... +[INFO] [1737513740.169287]: Video saved! +Starting episode 2 +[INFO] [1737513740.189427]: resetting environment +[INFO] [1737513740.202215]: cleared current plan +[INFO] [1737513740.204393]: moving to home +[INFO] [1737513743.406778]: reached home +[INFO] [1737513743.412605]: reset action history +[INFO] [1737513746.185742]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513750.274642]: resetting environment +[INFO] [1737513750.281453]: cleared current plan +[INFO] [1737513750.282225]: moving to home +[INFO] [1737513751.283941]: reached home +[INFO] [1737513751.284769]: reset action history +[INFO] [1737513754.044677]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/episode_rosbags/episode_2_2025-01-21-21-42-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2654253.log +started roslaunch server http://158.130.50.37:33147/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2654821] +process[print_text-8]: started with pid [2654822] +process[rosbag_record-9]: started with pid [2654823] +started bagging! +For topic gripper_width: timestamp difference is 100141321 for nearest: 1737513765279126917 - target: 1737513765178985596 at idx 158 +preempting episode +[INFO] [1737513769.366474]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1737513770.142069]: Stored episode 3. +[INFO] [1737513770.147829]: Saving video... +[INFO] [1737513770.437182]: Video saved! +Starting episode 3 +[INFO] [1737513770.477730]: resetting environment +[INFO] [1737513770.497528]: cleared current plan +[INFO] [1737513770.497926]: moving to home +[INFO] [1737513774.001151]: reached home +[INFO] [1737513774.007576]: reset action history +[INFO] [1737513776.732157]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513780.624867]: resetting environment +[INFO] [1737513780.640202]: cleared current plan +[INFO] [1737513780.640400]: moving to home +[INFO] [1737513781.641630]: reached home +[INFO] [1737513781.644108]: reset action history +[INFO] [1737513784.426559]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/episode_rosbags/episode_3_2025-01-21-21-43-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2654253.log +started roslaunch server http://158.130.50.37:42031/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2654962] +process[print_text-11]: started with pid [2654963] +process[rosbag_record-12]: started with pid [2654987] +started bagging! +For topic gripper_width: timestamp difference is 100014225 for nearest: 1737513806479107872 - target: 1737513806379093647 at idx 249 +[INFO] [1737513819.273411]: Storing episode... +[print_text-11] killing on exit[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1737513820.110896]: Stored episode 4. +[INFO] [1737513820.114705]: Saving video... +[INFO] [1737513820.505627]: Video saved! +Starting episode 4 +[INFO] [1737513820.545930]: resetting environment +[INFO] [1737513820.591512]: cleared current plan +[INFO] [1737513820.606401]: moving to home +[INFO] [1737513823.920047]: reached home +[INFO] [1737513823.929561]: reset action history +[INFO] [1737513826.713168]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513829.491615]: resetting environment +[INFO] [1737513829.510347]: cleared current plan +[INFO] [1737513829.510655]: moving to home +[INFO] [1737513830.513314]: reached home +[INFO] [1737513830.518350]: reset action history +[INFO] [1737513833.255994]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/episode_rosbags/episode_4_2025-01-21-21-43-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2654253.log +started roslaunch server http://158.130.50.37:41991/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2655103] +process[print_text-14]: started with pid [2655104] +process[rosbag_record-15]: started with pid [2655128] +started bagging! +preempting episode +[INFO] [1737513852.733987]: Storing episode... +[pub_text-13] killing on exit[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1737513853.547869]: Stored episode 5. +[INFO] [1737513853.554339]: Saving video... +[INFO] [1737513854.004261]: Video saved! +[INFO] [1737513854.077943]: resetting environment +[INFO] [1737513854.132822]: cleared current plan +[INFO] [1737513854.144043]: moving to home +[WARN] [1737513853.969538]: Plan exhausted +[WARN] [1737513854.013398]: Plan exhausted +[WARN] [1737513854.050550]: Plan exhausted +[WARN] [1737513854.087152]: Plan exhausted +[WARN] [1737513854.129694]: Plan exhausted +[INFO] [1737513857.560172]: reached home +[INFO] [1737513857.564023]: reset action history +[INFO] [1737513860.352352]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3763c61ce90250f1f6b6d05926da05e64f0be0a7 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:41:06.818942", + "startedAt": "2025-01-22T02:41:05.916233", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.31625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.154, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.356, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 398.9217071533203 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9f553f344b8d5ccf13d46f9337d411bd78a1b7da --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "42455952124d4dc1494a794ead57978b2c8afd8bbb95b5946273d6a77f3e4734", "size": 836964, "path": "media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4"}, "global_step": 399, "_timestamp": 1737513864.5986123, "_runtime": 198.67539620399475, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5d49fa2235834d91505b2ed986ef93f7c337e8bd15a86c32de0ac9065021b739", "size": 868076, "path": "media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4"}, "eval/num_success": 0.0, "episode": 30.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "d59a19bbca09918743b4ca42425de42894165a00081fd4e7f080290d9a85ab51", "size": 784931, "path": "media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "41dd8f6943cd079f1ca457d10191d663e4348f9fde2513ed4b2cd849d37ca46e", "size": 1510000, "path": "media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "812bd740546f7b5fae32b425bca424d6dff0d99fc3c07a9e3a3bc7d001cd30b7", "size": 937071, "path": "media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://jgn8zenlruskwacwqbt6kws0t6nktu2pffc6e8aiytql77q8xx2t8u1v9omhgvgn1yvtcs0joiwcnu6s7pnpx9oi84kziqindctga0q531gqbqu0rgonok0t48jid7zi/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://uld0w9btz1yo1gs4o6uo9aftgwfugxl7k5rz68f9kz0eqz0nb0rb96g6f1i1abqe3qs9w3fdowvwp599qf54bcutith0rdc0og4c7vg3y05nak6ljwjnwvw31yhxm3i6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 8, "total_num_episodes": 30, "total_success_rate": 0.26666666666666666, "total_success_rate_lb_0.01": 0.10985761888677481, "total_success_rate_ub_0.01": 0.4807906448879513, "total_success_rate_lb_0.025": 0.13731220371824376, "total_success_rate_ub_0.025": 0.45738425851720543, "total_success_rate_lb_0.05": 0.15524378425817548, "total_success_rate_ub_0.05": 0.4084890708510316, "total_success_rate_lb_0.1": 0.1665180363427985, "total_success_rate_ub_0.1": 0.39319007100810954, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "cfbe4ff6e2f3d3371ef324e1db9682569bf35f5835d1a5079cd1142b0a23cbb5", "size": 525, "artifact_path": "wandb-client-artifact://miv00vlx37gjyb51oy0643kbgzjku6j9ylzu36pkgpchpk9sb2fgv9le1vn6aefcumyj7zjgf1sd5q7bg6a79gaduuiifw9ivn24f9mr5ghhscbx65iltyrpj9nwsewi/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://n68jf0fps5mp8cz6ipwa2va1a0px4sbk58oft92rqg7wm9mkxow7v2w2nwi10bjpi07nh7ztvlg2fb2f1xf9svdub74638mgh9wz7iw7pukehg4orn704oinhou5i5p3:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 214}, "eval/total_success_rate": 0.2666666805744171, "eval/total_success_rate_lb_0.01": 0.10985761880874634, "eval/total_success_rate_ub_0.01": 0.4807906448841095, "eval/total_success_rate_lb_0.025": 0.13731220364570618, "eval/total_success_rate_ub_0.025": 0.45738425850868225, "eval/total_success_rate_lb_0.05": 0.15524378418922424, "eval/total_success_rate_ub_0.05": 0.40848907828330994, "eval/total_success_rate_lb_0.1": 0.16651803255081177, "eval/total_success_rate_ub_0.1": 0.39319008588790894} \ No newline at end of file diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug-internal.log b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6ca8656e8b3ba0daeb489ea06d6180019c145e9e --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug-internal.log @@ -0,0 +1,694 @@ +2025-01-21 21:41:05,923 INFO StreamThr :2654427 [internal.py:wandb_internal():85] W&B internal server running at pid: 2654427, started at: 2025-01-21 21:41:05.923231 +2025-01-21 21:41:05,925 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:41:05,935 INFO WriterThread:2654427 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb +2025-01-21 21:41:05,935 DEBUG SenderThread:2654427 [sender.py:send():379] send: header +2025-01-21 21:41:05,943 DEBUG SenderThread:2654427 [sender.py:send():379] send: run +2025-01-21 21:41:06,732 INFO SenderThread:2654427 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files +2025-01-21 21:41:06,732 INFO SenderThread:2654427 [sender.py:_start_run_threads():1188] run started: 8wyk5kwf with start time 1737513665.923216 +2025-01-21 21:41:06,751 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:41:06,751 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:41:06,800 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:41:06,805 DEBUG HandlerThread:2654427 [system_info.py:__init__():26] System info init +2025-01-21 21:41:06,805 DEBUG HandlerThread:2654427 [system_info.py:__init__():41] System info init done +2025-01-21 21:41:06,805 INFO HandlerThread:2654427 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:41:06,805 INFO SystemMonitor:2654427 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:41:06,805 INFO HandlerThread:2654427 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:41:06,806 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:41:06,806 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:41:06,807 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:41:06,807 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:41:06,808 INFO SystemMonitor:2654427 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:41:06,818 DEBUG HandlerThread:2654427 [system_info.py:probe():152] Probing system +2025-01-21 21:41:06,820 DEBUG HandlerThread:2654427 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:probe():200] Probing system done +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:41:06.818942', 'startedAt': '2025-01-22T02:41:05.916233', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.31625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.154, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.356, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 398.9217071533203}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:41:06,825 INFO HandlerThread:2654427 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:41:06,825 INFO HandlerThread:2654427 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:41:06,825 DEBUG HandlerThread:2654427 [system_info.py:_save_code():44] Saving code +2025-01-21 21:41:06,826 DEBUG HandlerThread:2654427 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:41:06,826 DEBUG HandlerThread:2654427 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:41:06,888 DEBUG HandlerThread:2654427 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:41:06,889 INFO HandlerThread:2654427 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:41:06,892 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:41:06,892 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:41:07,123 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/oziqii6d-wandb-metadata.json +2025-01-21 21:41:07,133 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:41:07,133 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:41:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:07,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:07,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:07,173 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/aajryftu-diff.patch +2025-01-21 21:41:07,177 INFO HandlerThread:2654427 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/tb" + save: true +} +control { + relay_id: "20885cd1050a4c509835d88f39296cfe" +} +_info { + stream_id: "8wyk5kwf" +} + +2025-01-21 21:41:07,177 DEBUG HandlerThread:2654427 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:41:07,179 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/gvff86vm-code/FISH/eval_robot.py +2025-01-21 21:41:07,287 DEBUG SenderThread:2654427 [sender.py:send():379] send: telemetry +2025-01-21 21:41:07,287 DEBUG SenderThread:2654427 [sender.py:send():379] send: tbrecord +2025-01-21 21:41:07,288 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:07,288 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 with policy live +2025-01-21 21:41:07,503 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/9mxep34g-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:07,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:07,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code +2025-01-21 21:41:07,734 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH +2025-01-21 21:41:08,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:09,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:09,733 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:10,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:11,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:11,288 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:12,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:13,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:14,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:15,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:16,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:16,880 DEBUG SenderThread:2654427 [sender.py:send():379] send: config +2025-01-21 21:41:16,880 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:17,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:18,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:19,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:19,735 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:20,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:21,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:21,736 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:22,133 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:22,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:22,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:22,227 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:23,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:23,736 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:24,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:25,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:25,737 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:26,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:27,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:27,738 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:28,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:28,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:29,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:29,738 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:30,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:31,137 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:31,739 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:32,138 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:33,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:33,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:34,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:35,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:36,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:37,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:37,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:38,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:38,199 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:38,741 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:41:39,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:40,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:41,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:42,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:43,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:44,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:44,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:45,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:46,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:47,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:47,677 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:41:47,678 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 with policy now +2025-01-21 21:41:47,678 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:41:47,679 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:41:47,679 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:41:47,681 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:41:47,743 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:41:47,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos +2025-01-21 21:41:47,900 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/j8bg8nph-media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:48,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:48,175 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/a9mud29r-media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:41:49,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:49,697 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:49,744 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:50,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:51,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:52,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:41:52,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:41:52,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:53,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:53,745 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:54,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:54,953 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:41:55,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:55,746 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:41:56,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:57,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:57,747 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:41:58,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:59,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:41:59,747 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:00,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:00,269 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:01,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:02,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:03,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:03,749 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:04,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:05,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:06,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:06,258 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:06,808 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:42:06,809 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:42:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:07,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:07,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:08,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:09,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:10,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:11,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:11,751 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:12,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:12,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:13,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:14,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:15,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:16,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:17,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:17,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:18,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:19,155 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:20,158 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:20,213 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:42:20,214 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 with policy now +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:20,214 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:20,215 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:20,444 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/3lrwmdyo-media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:20,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:42:21,042 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/ehwzovwb-media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:42:21,157 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:21,754 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:22,137 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:22,138 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:22,185 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:22,288 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:23,157 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:24,159 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:25,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:25,756 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:26,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:27,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:27,756 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:28,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:28,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:29,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:30,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:30,551 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:30,551 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:30,552 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:30,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:30,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:42:31,018 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/1urm2fut-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:42:31,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:31,758 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:32,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:33,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:33,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:33,759 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:34,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:35,160 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:35,759 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:36,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:36,809 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:42:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:37,134 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:37,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:38,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:38,286 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:39,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:40,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:41,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:42,161 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:43,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:44,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:44,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:45,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:46,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:47,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:47,763 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:48,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:49,139 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:49,163 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:50,167 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:50,481 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:42:50,481 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:42:50,482 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 with policy now +2025-01-21 21:42:50,482 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:42:50,482 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:42:50,482 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:42:50,710 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/vfahnxli-media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:50,764 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:42:50,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:50,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:42:51,048 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/zq6tkm2w-media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:42:51,170 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:51,765 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:52,140 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:42:52,141 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:42:52,185 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:53,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:54,170 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:55,045 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:42:55,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:55,766 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:56,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:57,169 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:57,767 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:42:58,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:42:59,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:00,141 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:00,171 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:00,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:01,179 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:01,371 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:01,372 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:01,372 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:01,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:01,768 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:02,179 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:03,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:03,769 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:04,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:05,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:05,684 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:05,769 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:06,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:06,810 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:43:07,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:07,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:07,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:08,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:09,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:10,172 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:11,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:11,302 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:12,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:13,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:14,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:15,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:16,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:17,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:17,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:18,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:19,173 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:20,174 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:21,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:22,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:22,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:22,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:22,292 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:23,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:24,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:25,175 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:26,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:27,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:27,777 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:28,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:28,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:29,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:30,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:31,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:32,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:33,142 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:33,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:34,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:35,176 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:36,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:36,810 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:43:37,134 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:37,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:37,181 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:38,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:38,202 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:39,177 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:40,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:40,549 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:43:40,549 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:43:40,550 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 with policy now +2025-01-21 21:43:40,550 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:40,550 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:40,576 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:40,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:43:40,800 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/h5tfg0fj-media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:41,100 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/3f5ebn25-media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:43:41,183 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:41,782 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:42,182 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:43,184 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:43,922 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:44,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:45,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:45,784 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:46,189 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:47,186 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:47,784 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:48,195 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:49,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:49,191 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:49,785 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:50,052 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/917ru0t5-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:43:50,190 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:50,455 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:43:50,455 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:43:50,455 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:43:50,785 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:43:51,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:51,786 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:52,138 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:43:52,138 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:43:52,199 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:53,196 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:54,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:54,531 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:43:55,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:55,787 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:43:56,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:57,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:58,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:43:59,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:00,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:00,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:01,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:02,192 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:03,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:04,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:05,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:06,143 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:06,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:06,811 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:07,135 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:07,135 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:07,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:08,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:09,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:10,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:11,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:11,299 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:12,193 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:13,200 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:13,793 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:14,072 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:14,072 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 with policy now +2025-01-21 21:44:14,077 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:14,077 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:14,077 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:14,078 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:14,197 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:14,310 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/u749tsej-media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:14,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval +2025-01-21 21:44:15,091 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/5qjme4xh-media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:15,201 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:15,794 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:16,202 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:16,795 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:17,148 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:17,204 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:18,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:19,206 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:19,796 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:20,223 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:21,206 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:21,796 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:22,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:22,144 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:22,216 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:22,299 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:23,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:23,797 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:24,099 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:44:24,099 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:44:24,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:24,298 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:24,619 INFO wandb-upload_1:2654427 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpi6fnlpzl +2025-01-21 21:44:24,678 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:44:24,688 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval +2025-01-21 21:44:24,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table +2025-01-21 21:44:25,052 INFO SenderThread:2654427 [sender.py:send_request_log_artifact():1525] logged artifact run-8wyk5kwf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4OTY0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5OTI1OQ==', 'latestArtifact': None}} +2025-01-21 21:44:25,052 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:25,052 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-21 21:44:25,052 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:25,053 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,066 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,066 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:25,066 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,084 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send():379] send: telemetry +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,085 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,085 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,107 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,107 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,127 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,127 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,128 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,128 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,151 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,151 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,173 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,173 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,195 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,195 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,196 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,196 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,209 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:25,217 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,217 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,236 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,236 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:25,242 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/mg9l7n33-media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:25,260 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:25,260 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:44:25,288 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/ztuohfpb-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:25,551 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/jcmpzxi2-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:25,700 INFO wandb-upload_0:2654427 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzqo7q131 +2025-01-21 21:44:25,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:25,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:26,210 INFO SenderThread:2654427 [sender.py:send_request_log_artifact():1525] logged artifact run-8wyk5kwf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4OTY3NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5OTI2NA==', 'latestArtifact': None}} +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send():379] send: files +2025-01-21 21:44:26,210 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json with policy now +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:26,210 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:26,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:26,232 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:26,491 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/t3s3dhil-media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:26,798 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:27,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:27,799 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:28,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:28,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:29,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:30,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:31,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:32,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:33,144 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:33,210 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:34,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:35,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:36,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:36,811 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:37,136 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:44:37,136 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:44:37,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:38,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:38,300 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:39,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:40,211 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:41,212 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:41,553 DEBUG SenderThread:2654427 [sender.py:send():379] send: exit +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():588] handling runtime: 214 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_exit():594] send defer +2025-01-21 21:44:41,554 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,554 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:44:41,554 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:44:41,554 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:44:41,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:44:41,555 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,555 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:44:41,555 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:44:41,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:44:41,555 INFO HandlerThread:2654427 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:44:41,555 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:44:41,555 DEBUG SystemMonitor:2654427 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:44:41,556 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:44:41,556 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:44:41,609 INFO HandlerThread:2654427 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:44:41,610 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,610 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:44:41,610 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:44:41,610 DEBUG SenderThread:2654427 [sender.py:send():379] send: stats +2025-01-21 21:44:41,610 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,610 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:44:41,611 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:41,611 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:44:41,611 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:44:41,611 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:41,611 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:44:41,805 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:43,806 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:48,502 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:48,502 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:44:48,502 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:44:48,502 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:44:48,503 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:48,503 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,503 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:44:48,504 DEBUG SenderThread:2654427 [sender.py:send():379] send: history +2025-01-21 21:44:48,504 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:44:48,505 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:48,506 DEBUG SenderThread:2654427 [sender.py:send():379] send: summary +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:44:48,506 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:44:48,506 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:44:48,506 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,506 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:44:48,507 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,507 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:44:48,510 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:44:48,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:48,656 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:44:48,656 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:48,656 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:48,656 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:44:48,656 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:48,656 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:44:48,808 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:48,808 INFO Thread-12 :2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:44:49,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:50,006 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:44:50,006 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:50,007 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,007 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:44:50,007 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,007 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:44:50,007 INFO SenderThread:2654427 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:44:50,009 INFO SenderThread:2654427 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:44:50,011 INFO SenderThread:2654427 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:44:50,021 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:44:50,021 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,021 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:44:50,021 DEBUG SenderThread:2654427 [sender.py:send():379] send: artifact +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:44:50,236 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:44:50,236 INFO SenderThread:2654427 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:44:50,555 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml config.yaml +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:50,809 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json wandb-summary.json +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt requirements.txt +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:44:50,811 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log output.log +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/diff.patch diff.patch +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 media/videos/eval/1_eval_1_5d49fa2235834d91505b.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 media/videos/eval/2_eval_2_d59a19bbca09918743b4.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 media/videos/eval/3_eval_3_41dd8f6943cd079f1ca4.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 media/videos/eval/0_eval_0_42455952124d4dc1494a.mp4 +2025-01-21 21:44:50,812 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 media/videos/eval/4_eval_4_812bd740546f7b5fae32.mp4 +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json media/table/eval/total_success_rate_ci_6_cfbe4ff6e2f3d3371ef3.table.json +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 21:44:50,814 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:44:50,814 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:50,817 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:50,817 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:44:50,817 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:50,817 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:44:50,817 INFO SenderThread:2654427 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:44:51,009 INFO wandb-upload_1:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/wandb-summary.json +2025-01-21 21:44:51,037 INFO wandb-upload_0:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/config.yaml +2025-01-21 21:44:51,062 INFO wandb-upload_2:2654427 [upload_job.py:push():130] Uploaded file /tmp/tmpgbnmm7_owandb/w8jxwy3h-events.out.tfevents.1737513667.leonmkim-ROG-Strix-G15CS-G15CS.2654253.0 +2025-01-21 21:44:51,072 INFO wandb-upload_3:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/requirements.txt +2025-01-21 21:44:51,083 INFO wandb-upload_4:2654427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/files/output.log +2025-01-21 21:44:51,283 INFO Thread-11 (_thread_body):2654427 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:44:51,283 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,283 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:44:51,283 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,283 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:44:51,284 INFO SenderThread:2654427 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:44:51,285 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,285 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:44:51,285 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:44:51,285 INFO SenderThread:2654427 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:44:51,490 INFO SenderThread:2654427 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:44:51,490 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:44:51,490 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,491 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,491 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:44:51,491 INFO SenderThread:2654427 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:44:51,491 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:44:51,491 INFO HandlerThread:2654427 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send():379] send: final +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send():379] send: footer +2025-01-21 21:44:51,491 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: defer +2025-01-21 21:44:51,492 INFO SenderThread:2654427 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:51,492 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:44:51,492 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:44:51,493 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:44:51,494 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:44:51,495 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:44:51,495 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:44:51,544 DEBUG SenderThread:2654427 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:44:51,544 INFO MainThread:2654427 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:44:51,545 INFO MainThread:2654427 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:44:51,545 INFO MainThread:2654427 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:44:51,545 DEBUG HandlerThread:2654427 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:44:51,545 INFO HandlerThread:2654427 [handler.py:finish():882] shutting down handler +2025-01-21 21:44:52,493 INFO WriterThread:2654427 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [sender.py:finish():1615] shutting down sender +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:44:52,544 INFO SenderThread:2654427 [file_pusher.py:join():175] waiting for file pusher diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug.log b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7b91939e67b993ceb3e08c7f0820951807923eb2 --- /dev/null +++ b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Configure stats pid to 2654253 +2025-01-21 21:41:05,917 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/settings +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug.log +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/logs/debug-internal.log +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf'} +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():619] starting backend +2025-01-21 21:41:05,918 INFO MainThread:2654253 [wandb_init.py:init():623] setting up manager +2025-01-21 21:41:05,922 INFO MainThread:2654253 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:41:05,923 INFO MainThread:2654253 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:41:05,934 INFO MainThread:2654253 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:41:05,941 INFO MainThread:2654253 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:41:06,750 INFO MainThread:2654253 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:41:06,795 INFO MainThread:2654253 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:41:06,795 INFO MainThread:2654253 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:41:07,134 INFO MainThread:2654253 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:41:07,135 INFO MainThread:2654253 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:41:07,137 INFO MainThread:2654253 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:41:07,137 INFO MainThread:2654253 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/8wyk5kwf/tb, True +2025-01-21 21:41:16,879 INFO MainThread:2654253 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:44:52,546 WARNING MsgRouterThr:2654253 [router.py:message_loop():77] message_loop has been closed diff --git a/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..abdf6bfe3b49ef7e84fb40061f2504559c095abd Binary files /dev/null and b/8wyk5kwf/wandb/run-20250121_214105-8wyk5kwf/run-8wyk5kwf.wandb differ diff --git a/aqyye28j/episode_rosbags/aligned_depth_to_color_K.npy b/aqyye28j/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aqyye28j/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aqyye28j/episode_rosbags/cam_tf_world.npy b/aqyye28j/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/aqyye28j/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/aqyye28j/episode_rosbags/color_K.npy b/aqyye28j/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aqyye28j/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aqyye28j/episode_rosbags/depth_K.npy b/aqyye28j/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/aqyye28j/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/aqyye28j/eval_robot.log b/aqyye28j/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..856f8c5431c9101470c3dd3690af40aaccbe6b81 --- /dev/null +++ b/aqyye28j/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 22:07:02,378][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:07:02,381][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/aqyye28j/wandb/debug-internal.log b/aqyye28j/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3ba7297175f55ecba0d939a4bd57721161c2de77 --- /dev/null +++ b/aqyye28j/wandb/debug-internal.log @@ -0,0 +1,759 @@ +2025-01-21 22:07:08,229 INFO StreamThr :2661581 [internal.py:wandb_internal():85] W&B internal server running at pid: 2661581, started at: 2025-01-21 22:07:08.229643 +2025-01-21 22:07:08,231 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:07:08,242 INFO WriterThread:2661581 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb +2025-01-21 22:07:08,243 DEBUG SenderThread:2661581 [sender.py:send():379] send: header +2025-01-21 22:07:08,252 DEBUG SenderThread:2661581 [sender.py:send():379] send: run +2025-01-21 22:07:08,592 INFO SenderThread:2661581 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files +2025-01-21 22:07:08,592 INFO SenderThread:2661581 [sender.py:_start_run_threads():1188] run started: aqyye28j with start time 1737515228.229771 +2025-01-21 22:07:08,607 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:07:08,607 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:07:08,686 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:07:08,691 DEBUG HandlerThread:2661581 [system_info.py:__init__():26] System info init +2025-01-21 22:07:08,691 DEBUG HandlerThread:2661581 [system_info.py:__init__():41] System info init done +2025-01-21 22:07:08,691 INFO HandlerThread:2661581 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:07:08,691 INFO SystemMonitor:2661581 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:07:08,691 INFO HandlerThread:2661581 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:07:08,692 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:07:08,692 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:07:08,705 DEBUG HandlerThread:2661581 [system_info.py:probe():152] Probing system +2025-01-21 22:07:08,706 DEBUG HandlerThread:2661581 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:probe():200] Probing system done +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:07:08.705289', 'startedAt': '2025-01-22T03:07:08.219659', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.8076249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.981, 'min': 800.0, 'max': 4700.0}, {'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 449.60555267333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:07:08,711 INFO HandlerThread:2661581 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:07:08,711 INFO HandlerThread:2661581 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:_save_code():44] Saving code +2025-01-21 22:07:08,713 DEBUG HandlerThread:2661581 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:07:08,713 DEBUG HandlerThread:2661581 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:07:08,776 DEBUG HandlerThread:2661581 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:07:08,776 INFO HandlerThread:2661581 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:07:08,780 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:07:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:07:09,018 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:07:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:09,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:09,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:09,022 INFO HandlerThread:2661581 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/tb" + save: true +} +control { + relay_id: "75b487f69feb46dca4f1836e763f05bd" +} +_info { + stream_id: "aqyye28j" +} + +2025-01-21 22:07:09,022 DEBUG HandlerThread:2661581 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:07:09,047 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/7pk3fx6n-wandb-metadata.json +2025-01-21 22:07:09,076 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/77h50yn7-code/FISH/eval_robot.py +2025-01-21 22:07:09,079 DEBUG SenderThread:2661581 [sender.py:send():379] send: telemetry +2025-01-21 22:07:09,079 DEBUG SenderThread:2661581 [sender.py:send():379] send: tbrecord +2025-01-21 22:07:09,080 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:07:09,080 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 with policy live +2025-01-21 22:07:09,088 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/6z2xrcrr-diff.patch +2025-01-21 22:07:09,277 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/gr073swi-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code +2025-01-21 22:07:09,595 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH +2025-01-21 22:07:10,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:11,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:11,593 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:12,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:13,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:14,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:14,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:15,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:16,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:16,715 DEBUG SenderThread:2661581 [sender.py:send():379] send: config +2025-01-21 22:07:17,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:18,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:19,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:19,595 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:20,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:20,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:21,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:21,596 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:22,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:23,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:23,597 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:24,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:24,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:25,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:25,209 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:25,597 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:26,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:27,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:28,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:29,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:30,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:31,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:31,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:31,599 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:32,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:33,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:33,600 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:34,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:35,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:35,600 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:36,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:36,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:37,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:38,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:39,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:39,018 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:39,601 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:40,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:41,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:41,162 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:41,602 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:07:42,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:43,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:44,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:45,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:46,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:47,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:47,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:48,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:49,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:50,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:51,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:52,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:53,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:53,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:54,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:54,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:54,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:55,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:56,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:57,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:58,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:58,181 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:59,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:00,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:01,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:02,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:03,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:04,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:04,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:05,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:06,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:07,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:08,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:08,694 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:08:08,694 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:08:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:09,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:09,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:09,165 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:09,826 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:08:09,827 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 with policy now +2025-01-21 22:08:09,827 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:08:09,828 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:09,828 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:09,829 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:10,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:10,109 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/8b7e7wby-media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:10,609 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:10,609 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:08:10,924 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/2it6cc87-media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:11,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:11,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:12,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:13,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:14,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:15,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:15,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:15,611 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:16,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:17,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:17,612 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:18,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:19,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:20,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:20,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:21,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:21,613 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:21,877 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/xjglzqfs-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:22,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:23,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:23,614 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:24,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:24,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:25,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:25,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:26,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:27,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:27,615 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:28,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:29,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:30,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:31,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:31,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:32,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:33,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:34,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:35,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:36,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:36,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:37,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:38,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:38,695 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:08:39,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:39,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:40,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:41,041 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:41,284 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:41,443 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:08:41,443 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 with policy now +2025-01-21 22:08:41,446 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:08:41,446 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:41,446 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:41,447 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:08:41,703 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/ifuepsg8-media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:41,975 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/8xh0el6j-media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:42,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:43,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:43,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:44,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:45,046 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:46,042 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:47,005 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:47,042 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:47,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:48,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:48,412 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:48,412 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:48,413 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:48,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:48,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:49,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:49,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:50,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:51,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:51,623 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:52,007 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:52,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:53,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:53,624 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:54,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:54,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:54,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:55,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:56,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:57,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:57,162 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:58,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:59,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:00,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:01,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:01,626 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:02,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:03,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:03,125 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:04,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:05,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:06,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:07,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:07,529 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:09:07,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 with policy now +2025-01-21 22:09:07,530 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:09:07,530 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:07,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:07,533 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:07,628 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:09:07,800 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/em1fn7ja-media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:08,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:08,064 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/xkxneh5e-media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:08,560 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:08,695 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:09:09,036 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:09,037 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:09,081 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:09,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:10,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:11,059 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:12,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:13,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:13,630 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:13,806 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:14,048 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:15,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:15,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:09:15,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:16,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:16,224 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:16,224 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:16,225 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:16,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:17,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:17,632 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:18,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:19,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:19,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:20,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:21,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:21,633 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:22,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:23,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:24,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:24,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:25,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:26,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:27,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:28,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:29,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:30,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:30,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:31,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:32,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:33,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:34,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:35,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:35,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:36,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:37,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:38,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:38,696 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:09:39,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:39,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:40,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:40,101 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:41,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:42,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:43,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:44,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:45,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:46,027 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:46,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:47,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:48,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:49,055 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:49,642 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:50,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:51,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:51,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:52,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:53,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:54,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:54,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:54,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:55,057 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:56,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:56,411 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:56,618 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:09:56,618 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 with policy now +2025-01-21 22:09:56,624 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:09:56,624 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:56,624 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:56,632 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:09:56,917 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/ptqyh5vo-media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:57,058 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:57,214 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/d819ww7a-media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:57,646 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:58,062 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:59,075 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:00,064 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:01,063 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:01,648 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:02,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:02,317 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:03,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:04,064 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:04,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:04,910 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/fdx00fwz-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:05,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:05,278 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:05,278 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:05,278 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:05,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:05,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:06,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:07,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:07,650 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:07,697 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:08,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:08,696 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:10:09,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:09,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:09,066 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:09,651 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:10,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:11,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:11,651 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:12,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:13,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:13,115 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:14,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:15,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:16,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:17,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:18,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:19,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:19,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:20,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:21,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:22,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:23,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:24,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:24,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:24,145 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:25,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:25,656 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:26,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:27,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:27,657 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:28,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:29,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:29,658 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:30,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:30,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:31,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:32,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:33,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:34,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:35,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:35,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:36,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:37,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:37,660 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:38,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:38,697 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:10:39,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:39,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:39,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:40,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:40,143 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:41,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:42,072 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:43,072 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:44,075 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:44,859 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:44,859 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 with policy now +2025-01-21 22:10:44,861 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:44,862 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:44,862 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:44,866 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:45,073 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:45,129 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/wwebsqsd-media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:45,663 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:45,663 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:45,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:45,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:10:45,888 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:45,976 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/koso4oum-media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:46,081 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:47,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:47,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:48,078 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:49,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:49,665 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:50,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:51,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:51,099 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:52,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:53,088 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:53,666 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:54,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:54,034 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:54,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:55,097 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:56,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:56,192 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:57,089 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:57,343 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:57,343 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:57,345 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:57,589 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:10:57,589 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval +2025-01-21 22:10:57,789 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:57,863 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/1x4ld4vr-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:58,071 INFO wandb-upload_0:2661581 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0d7hi5di +2025-01-21 22:10:58,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:58,236 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:10:58,247 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:58,522 INFO SenderThread:2661581 [sender.py:send_request_log_artifact():1525] logged artifact run-aqyye28j-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyMDg0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNDgzMg==', 'latestArtifact': None}} +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:58,522 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,523 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,523 DEBUG SenderThread:2661581 [sender.py:send():379] send: telemetry +2025-01-21 22:10:58,523 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,527 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,527 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,528 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,528 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,528 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,528 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,529 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,529 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,530 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,530 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,530 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,530 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,531 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,531 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,532 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,532 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,532 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,532 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:58,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval +2025-01-21 22:10:58,786 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/6e4upphm-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:58,861 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/64lup3tl-media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:58,946 INFO wandb-upload_1:2661581 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpztmwj91e +2025-01-21 22:10:59,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:59,350 INFO SenderThread:2661581 [sender.py:send_request_log_artifact():1525] logged artifact run-aqyye28j-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyMDg2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNDgzOA==', 'latestArtifact': None}} +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:59,350 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json with policy now +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:59,351 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:59,616 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/t5uoni3c-media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:59,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:59,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:00,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:01,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:01,351 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:02,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:03,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:04,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:05,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:06,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:07,033 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:07,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:08,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:08,697 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:11:09,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:11:09,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:11:09,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:10,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:11,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:12,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:12,148 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:13,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:14,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:15,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:16,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:17,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:18,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:18,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:19,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:20,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:21,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:22,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:23,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:23,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:11:24,022 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:11:24,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:25,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:26,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:27,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:28,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:28,182 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:28,421 DEBUG SenderThread:2661581 [sender.py:send():379] send: exit +2025-01-21 22:11:28,421 INFO SenderThread:2661581 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:11:28,421 INFO SenderThread:2661581 [sender.py:send_exit():588] handling runtime: 259 +2025-01-21 22:11:28,422 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_exit():594] send defer +2025-01-21 22:11:28,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,423 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:11:28,423 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:11:28,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,423 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:11:28,423 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:11:28,424 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:11:28,424 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:11:28,424 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:11:28,477 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,477 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:11:28,477 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:11:28,478 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:11:28,478 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,478 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:11:28,478 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,478 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:11:28,478 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:11:28,478 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,478 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:11:28,681 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:11:28,682 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:29,682 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:35,352 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:35,353 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:35,353 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:11:35,354 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,354 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:11:35,354 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:11:35,354 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,354 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:11:35,354 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:11:35,355 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send():379] send: summary +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:35,356 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:11:35,356 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,356 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:11:35,356 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:11:35,356 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:35,357 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,357 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:11:35,357 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,357 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:11:35,422 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:35,460 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:11:35,461 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:35,461 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,461 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:11:35,461 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,461 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:11:35,461 INFO SenderThread:2661581 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:11:35,464 INFO SenderThread:2661581 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:11:35,466 INFO SenderThread:2661581 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:11:35,474 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:11:35,474 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,474 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:11:35,474 DEBUG SenderThread:2661581 [sender.py:send():379] send: artifact +2025-01-21 22:11:35,684 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:35,685 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:11:35,776 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:11:36,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml config.yaml +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json wandb-summary.json +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt requirements.txt +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log output.log +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch diff.patch +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:11:36,691 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:36,691 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:36,692 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:11:36,692 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:36,692 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:11:36,692 INFO SenderThread:2661581 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:11:36,881 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:36,918 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt +2025-01-21 22:11:36,936 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:11:36,940 INFO wandb-upload_4:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:37,007 INFO wandb-upload_3:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/k5suzsul-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:11:37,207 INFO Thread-11 (_thread_body):2661581 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:11:37,207 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,208 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:11:37,208 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,208 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:11:37,208 INFO SenderThread:2661581 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:11:37,211 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:11:37,211 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,211 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:11:37,211 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,212 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:11:37,212 INFO SenderThread:2661581 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:11:37,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,429 INFO SenderThread:2661581 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:11:37,429 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:11:37,429 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,429 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,429 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:11:37,429 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:11:37,430 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,430 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send():379] send: final +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send():379] send: footer +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:11:37,430 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,431 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:11:37,431 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,431 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,431 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:11:37,433 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:11:37,433 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:11:37,434 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:37,485 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:11:37,487 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:11:37,487 INFO HandlerThread:2661581 [handler.py:finish():882] shutting down handler +2025-01-21 22:11:38,433 INFO WriterThread:2661581 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [sender.py:finish():1615] shutting down sender +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [file_pusher.py:join():175] waiting for file pusher diff --git a/aqyye28j/wandb/debug.log b/aqyye28j/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1ea1559c0ff3f54d8a9ff0734f59dca5bfcd21f0 --- /dev/null +++ b/aqyye28j/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Configure stats pid to 2661386 +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/settings +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug.log +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug-internal.log +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j'} +2025-01-21 22:07:08,224 INFO MainThread:2661386 [wandb_init.py:init():619] starting backend +2025-01-21 22:07:08,224 INFO MainThread:2661386 [wandb_init.py:init():623] setting up manager +2025-01-21 22:07:08,228 INFO MainThread:2661386 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:07:08,229 INFO MainThread:2661386 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:07:08,240 INFO MainThread:2661386 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:07:08,250 INFO MainThread:2661386 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:07:08,606 INFO MainThread:2661386 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:07:08,679 INFO MainThread:2661386 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:07:08,679 INFO MainThread:2661386 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:07:09,022 INFO MainThread:2661386 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:07:09,022 INFO MainThread:2661386 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/tb, True +2025-01-21 22:07:16,714 INFO MainThread:2661386 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:11:38,487 WARNING MsgRouterThr:2661386 [router.py:message_loop():77] message_loop has been closed diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..478228d798aae13171c243beceeb64fbf285cdd2 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737515228 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]} \ No newline at end of file diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..290582fb2214e70db6001b483dbeea91a135af4b --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[11, 35, 0.3142857142857143, 0.15367382771742805, 0.5138078033896487, 0.1823880747595864, 0.4915851503627442, 0.20139703905594025, 0.4481028467520558, 0.2145629972790341, 0.4322889745370866]]} \ No newline at end of file diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07ab74cf9dbce72408e70009d4cc9b7a06b7c67f --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51125b9dabf4e9b3cedcd36bf6f097b605281070efe383308692d343bdc07522 +size 1526251 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb65a48e3a8f1d48d74184448288e387f725d204 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27250e24fb2dabe74cd6138f841e48b5062f21ab0105d92770e2a1dc76782e8f +size 777927 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b00ff0dbe184831ac7b4301aaa3b7c4e3821cfb --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96f346beb19c014ed98ce09ddc37c0e6bc4a2ec48377abfbcdedd9e65e8ed485 +size 744132 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7d00b2bb2d3fab2aa7a2927051d5289c63c1b72 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cfd6f1db2c9dcc9eb14fdd05e29177fb84a17d1241df2bccda3ec99c472c710 +size 1566794 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1eaad92644d6641719d8472b7287cc0f37003411 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log @@ -0,0 +1,288 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737515236.715368]: resetting environment +[INFO] [1737515236.722792]: cleared current plan +[INFO] [1737515236.724116]: moving to home +[INFO] [1737515237.725321]: reached home +[INFO] [1737515237.726756]: reset action history +[INFO] [1737515240.436699]: environment reset +Starting episode 0 +[INFO] [1737515240.441116]: resetting environment +[INFO] [1737515240.448637]: cleared current plan +[INFO] [1737515240.451077]: moving to home +[INFO] [1737515241.454871]: reached home +[INFO] [1737515241.456529]: reset action history +[INFO] [1737515244.166986]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737515250.010605]: resetting environment +[INFO] [1737515250.015251]: cleared current plan +[INFO] [1737515250.016377]: moving to home +[INFO] [1737515251.018677]: reached home +[INFO] [1737515251.019852]: reset action history +[INFO] [1737515253.728170]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/episode_rosbags/episode_0_2025-01-21-22-07-33.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2661386.log +started roslaunch server http://158.130.50.37:36123/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2661690] +process[print_text-2]: started with pid [2661714] +process[rosbag_record-3]: started with pid [2661715] +started bagging! +For topic gripper_width: timestamp difference is 103357874 for nearest: 1737515257112467609 - target: 1737515257009109735 at idx 52 +For topic gripper_width: timestamp difference is -106109824 for nearest: 1737515257712465558 - target: 1737515257818575382 at idx 59 +[INFO] [1737515288.781087]: Storing episode... +[print_text-2] killing on exit[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1737515289.590624]: Stored episode 1. +[INFO] [1737515289.590833]: Saving video... +[INFO] [1737515289.816034]: Video saved! +Starting episode 1 +[INFO] [1737515289.829769]: resetting environment +[INFO] [1737515289.841089]: cleared current plan +[INFO] [1737515289.841413]: moving to home +[INFO] [1737515293.444704]: reached home +[INFO] [1737515293.445143]: reset action history +[INFO] [1737515296.156034]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515301.053850]: resetting environment +[INFO] [1737515301.061250]: cleared current plan +[INFO] [1737515301.061623]: moving to home +[INFO] [1737515302.063107]: reached home +[INFO] [1737515302.064437]: reset action history +[INFO] [1737515304.772004]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/episode_rosbags/episode_1_2025-01-21-22-08-24.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2661386.log +started roslaunch server http://158.130.50.37:46561/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2661839] +process[print_text-5]: started with pid [2661840] +process[rosbag_record-6]: started with pid [2661841] +started bagging! +preempting episode +[INFO] [1737515320.085987]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1737515320.857978]: Stored episode 2. +[INFO] [1737515320.858253]: Saving video... +[INFO] [1737515321.422727]: Video saved! +Starting episode 2 +[INFO] [1737515321.448649]: resetting environment +[INFO] [1737515321.460191]: cleared current plan +[INFO] [1737515321.460438]: moving to home +[WARN] [1737515321.284052]: Plan exhausted +[WARN] [1737515321.323966]: Plan exhausted +[WARN] [1737515321.363975]: Plan exhausted +[WARN] [1737515321.404851]: Plan exhausted +[WARN] [1737515321.445599]: Plan exhausted +[INFO] [1737515324.963636]: reached home +[INFO] [1737515324.966800]: reset action history +[INFO] [1737515327.742065]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515328.023034]: resetting environment +[INFO] [1737515328.040882]: cleared current plan +[INFO] [1737515328.041167]: moving to home +[INFO] [1737515329.044321]: reached home +[INFO] [1737515329.045142]: reset action history +[INFO] [1737515331.757763]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/episode_rosbags/episode_2_2025-01-21-22-08-51.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2661386.log +started roslaunch server http://158.130.50.37:40117/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2661981] +process[print_text-8]: started with pid [2661982] +process[rosbag_record-9]: started with pid [2661983] +started bagging! +For topic gripper_width: timestamp difference is 106824790 for nearest: 1737515340079160129 - target: 1737515339972335339 at idx 112 +preempting episode +[INFO] [1737515346.376017]: Storing episode... +[pub_text-7] killing on exit[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1737515347.187423]: Stored episode 3. +[INFO] [1737515347.188803]: Saving video... +[INFO] [1737515347.507875]: Video saved! +Starting episode 3 +[INFO] [1737515347.536609]: resetting environment +[INFO] [1737515347.559703]: cleared current plan +[INFO] [1737515347.560046]: moving to home +[INFO] [1737515351.065438]: reached home +[INFO] [1737515351.067636]: reset action history +[INFO] [1737515353.804307]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515355.492930]: resetting environment +[INFO] [1737515355.517884]: cleared current plan +[INFO] [1737515355.521447]: moving to home +[INFO] [1737515356.529325]: reached home +[INFO] [1737515356.534803]: reset action history +[INFO] [1737515359.315569]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/episode_rosbags/episode_3_2025-01-21-22-09-19.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2661386.log +started roslaunch server http://158.130.50.37:40993/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2662144] +process[print_text-11]: started with pid [2662145] +process[rosbag_record-12]: started with pid [2662146] +started bagging! +For topic gripper_width: timestamp difference is 102464237 for nearest: 1737515388279162445 - target: 1737515388176698208 at idx 249 +[INFO] [1737515395.059456]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737515395.946960]: Stored episode 4. +[INFO] [1737515395.948910]: Saving video... +[INFO] [1737515396.580140]: Video saved! +Starting episode 4 +[INFO] [1737515396.634619]: resetting environment +[INFO] [1737515396.659529]: cleared current plan +[INFO] [1737515396.661899]: moving to home +[WARN] [1737515396.411053]: Plan exhausted +[WARN] [1737515396.451798]: Plan exhausted +[WARN] [1737515396.491116]: Plan exhausted +[WARN] [1737515396.535413]: Plan exhausted +[WARN] [1737515396.570312]: Plan exhausted +[WARN] [1737515396.610757]: Plan exhausted +[WARN] [1737515396.650353]: Plan exhausted +[INFO] [1737515400.271382]: reached home +[INFO] [1737515400.273765]: reset action history +[INFO] [1737515403.018003]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515404.619018]: resetting environment +[INFO] [1737515404.642256]: cleared current plan +[INFO] [1737515404.644649]: moving to home +[INFO] [1737515405.653596]: reached home +[INFO] [1737515405.658616]: reset action history +[INFO] [1737515408.397864]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/episode_rosbags/episode_4_2025-01-21-22-10-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2661386.log +started roslaunch server http://158.130.50.37:34765/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2662285] +process[print_text-14]: started with pid [2662286] +process[rosbag_record-15]: started with pid [2662287] +started bagging! +For topic gripper_width: timestamp difference is 108770455 for nearest: 1737515424612467373 - target: 1737515424503696918 at idx 206 +For topic gripper_width: timestamp difference is 110366245 for nearest: 1737515425379134794 - target: 1737515425268768549 at idx 215 +For topic gripper_width: timestamp difference is 102234071 for nearest: 1737515427479160016 - target: 1737515427376925945 at idx 241 +For topic gripper_width: timestamp difference is 112466016 for nearest: 1737515434979135671 - target: 1737515434866669655 at idx 249 +[INFO] [1737515443.216545]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737515444.131016]: Stored episode 5. +[INFO] [1737515444.133822]: Saving video... +[INFO] [1737515444.820015]: Video saved! +[WARN] [1737515444.573115]: Plan exhausted +[WARN] [1737515444.610329]: Plan exhausted +[WARN] [1737515444.650401]: Plan exhausted +[WARN] [1737515444.690349]: Plan exhausted +[WARN] [1737515444.730970]: Plan exhausted +[WARN] [1737515444.772886]: Plan exhausted +[WARN] [1737515444.813456]: Plan exhausted +[WARN] [1737515444.853793]: Plan exhausted +[INFO] [1737515444.862858]: resetting environment +[INFO] [1737515444.887275]: cleared current plan +[INFO] [1737515444.887521]: moving to home +[INFO] [1737515448.492999]: reached home +[INFO] [1737515448.499923]: reset action history +[INFO] [1737515451.318604]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cf18756aa2c1d388ddee766f8e9f5a244cfdf214 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T03:07:08.705289", + "startedAt": "2025-01-22T03:07:08.219659", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4123.8076249999995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.981, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.971, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 449.60555267333984 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6a47ed56c6762f3092b4b62171ae586c15ff5b --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "51125b9dabf4e9b3cedcd36bf6f097b605281070efe383308692d343bdc07522", "size": 1526251, "path": "media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4"}, "global_step": 512, "_timestamp": 1737515458.1354377, "_runtime": 229.90566682815552, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "27250e24fb2dabe74cd6138f841e48b5062f21ab0105d92770e2a1dc76782e8f", "size": 777927, "path": "media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4"}, "eval/num_success": 3.0, "episode": 35.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "96f346beb19c014ed98ce09ddc37c0e6bc4a2ec48377abfbcdedd9e65e8ed485", "size": 744132, "path": "media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "3cfd6f1db2c9dcc9eb14fdd05e29177fb84a17d1241df2bccda3ec99c472c710", "size": 1566794, "path": "media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6afd64f2a3f512f6d9a67af92605d237cd01284da9aacd680b56871000a9afbe", "size": 1532772, "path": "media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://xibbr9qpe58wde3wni2gbq3r5p2t5x75eyvbew05pux3pkenojalq6oom460ejcn1bjaa0c4akrjm7c1oh4jd0426crpv29gtm3roctdypgbirbj5hvtd9wgi7iwj1g6/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vt9apcspcfd6sfule0wrk0eh1ypqekrmafje6veq8toy0tvynvzisykaluc2y6ot06cu3h68cwdc8fzszfioilw6uuzkcs31m6kg96sk4620s3p21kojqmzfqg0vybig:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 11, "total_num_episodes": 35, "total_success_rate": 0.3142857142857143, "total_success_rate_lb_0.01": 0.15367382771742805, "total_success_rate_ub_0.01": 0.5138078033896487, "total_success_rate_lb_0.025": 0.1823880747595864, "total_success_rate_ub_0.025": 0.4915851503627442, "total_success_rate_lb_0.05": 0.20139703905594025, "total_success_rate_ub_0.05": 0.4481028467520558, "total_success_rate_lb_0.1": 0.2145629972790341, "total_success_rate_ub_0.1": 0.4322889745370866, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "41c18ac1da7a95c048c4e86e7f0f1808431e6b963187faa4c7dc79c01b89799c", "size": 522, "artifact_path": "wandb-client-artifact://rph7hewn133thnd532tvyv39u7mamgn0ue6vk3eglshwr8k2ug26yjzqwbshu7jd52t251sjyrbyjz9abxewtht6jqwjo0xmp1wvpdg1cmkr582snkfyh5766rvie85z/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://s24jv78jdgewj5grqwhk7js2clovv32c4w2x72e6joq36gulbmthmtwk4f4jz5geuz1j1qssrtmaio3devgkhdfndaywq1774ggrn973awibco0qa52z4kg051rn78ww:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 259}, "eval/total_success_rate": 0.3142857253551483, "eval/total_success_rate_lb_0.01": 0.15367382764816284, "eval/total_success_rate_ub_0.01": 0.5138077735900879, "eval/total_success_rate_lb_0.025": 0.1823880821466446, "eval/total_success_rate_ub_0.025": 0.4915851652622223, "eval/total_success_rate_lb_0.05": 0.20139704644680023, "eval/total_success_rate_ub_0.05": 0.4481028616428375, "eval/total_success_rate_lb_0.1": 0.21456299722194672, "eval/total_success_rate_ub_0.1": 0.4322889745235443} \ No newline at end of file diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug.log b/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1ea1559c0ff3f54d8a9ff0734f59dca5bfcd21f0 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Configure stats pid to 2661386 +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:07:08,222 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/settings +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug.log +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug-internal.log +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:07:08,223 INFO MainThread:2661386 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j'} +2025-01-21 22:07:08,224 INFO MainThread:2661386 [wandb_init.py:init():619] starting backend +2025-01-21 22:07:08,224 INFO MainThread:2661386 [wandb_init.py:init():623] setting up manager +2025-01-21 22:07:08,228 INFO MainThread:2661386 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:07:08,229 INFO MainThread:2661386 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:07:08,240 INFO MainThread:2661386 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:07:08,250 INFO MainThread:2661386 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:07:08,606 INFO MainThread:2661386 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:07:08,679 INFO MainThread:2661386 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:07:08,679 INFO MainThread:2661386 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:07:09,019 INFO MainThread:2661386 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:07:09,022 INFO MainThread:2661386 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:07:09,022 INFO MainThread:2661386 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/tb, True +2025-01-21 22:07:16,714 INFO MainThread:2661386 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:11:38,487 WARNING MsgRouterThr:2661386 [router.py:message_loop():77] message_loop has been closed diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb b/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb new file mode 100644 index 0000000000000000000000000000000000000000..740b6f5b46e361b775dad917e384ec336a8ec5eb Binary files /dev/null and b/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb differ diff --git a/hcp6xxlt/episode_rosbags/episode_0_2025-01-21-21-36-09.bag b/hcp6xxlt/episode_rosbags/episode_0_2025-01-21-21-36-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..3ba4419405e87bd26f5f68145a533d89893d4637 --- /dev/null +++ b/hcp6xxlt/episode_rosbags/episode_0_2025-01-21-21-36-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb5b3e500c01d84b782a3d1e3fad02ae942efcc8a87425f9c12fdc7f27e90a0 +size 1042101215 diff --git a/hcp6xxlt/episode_rosbags/episode_1_2025-01-21-21-36-35.bag b/hcp6xxlt/episode_rosbags/episode_1_2025-01-21-21-36-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..88cd2fef8d6dde1d347810975e07b5759c594ebd --- /dev/null +++ b/hcp6xxlt/episode_rosbags/episode_1_2025-01-21-21-36-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50331a8f4019e0a3d5449ef77711490c5ce4770bae37a2a6522bdd4f63e3131d +size 1184229100 diff --git a/hcp6xxlt/episode_rosbags/episode_2_2025-01-21-21-37-44.bag b/hcp6xxlt/episode_rosbags/episode_2_2025-01-21-21-37-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..5117b4ff2cc0ef1b94f5fb5ef12c1dc2a2a8f8e9 --- /dev/null +++ b/hcp6xxlt/episode_rosbags/episode_2_2025-01-21-21-37-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f53593b86e95c9a49b2b053d4c9e0acb0a2cd58d08b3d47464a70122c9a6dc +size 1023803731 diff --git a/m9j39xya/.hydra/config.yaml b/m9j39xya/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/m9j39xya/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/m9j39xya/.hydra/hydra.yaml b/m9j39xya/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1570cd3f9878c5dcb26542ca982df3a8727957af --- /dev/null +++ b/m9j39xya/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/m9j39xya/.hydra/overrides.yaml b/m9j39xya/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/m9j39xya/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/m9j39xya/episode_rosbags/aligned_depth_to_color_K.npy b/m9j39xya/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/m9j39xya/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/m9j39xya/episode_rosbags/cam_tf_world.npy b/m9j39xya/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/m9j39xya/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/m9j39xya/episode_rosbags/color_K.npy b/m9j39xya/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/m9j39xya/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/m9j39xya/episode_rosbags/depth_K.npy b/m9j39xya/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/m9j39xya/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/m9j39xya/eval_robot.log b/m9j39xya/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..5b826034f905f351491909474ca42dd8132ba04f --- /dev/null +++ b/m9j39xya/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 22:12:42,505][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:12:42,508][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/m9j39xya/eval_video/0_eval.mp4 b/m9j39xya/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0c60d96f1d5d1a5bf6422eeacb5dc361cca7e91 --- /dev/null +++ b/m9j39xya/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be41a6a3e5ef05ec42a0b86fd98317313bee30f6316ecc0cfb731364ee60638d +size 726574 diff --git a/m9j39xya/eval_video/1_eval.mp4 b/m9j39xya/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff6e6c31a2343624d224a56c6539ba1074fa98b8 --- /dev/null +++ b/m9j39xya/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad3604e3344632866d9dde11b11155716453c35ab19983ca5cc1ac428f91b64b +size 704030 diff --git a/m9j39xya/eval_video/2_eval.mp4 b/m9j39xya/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..347581392d8131a045fae763c6a09108cc99d25b --- /dev/null +++ b/m9j39xya/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11990a56a7de0e8d16ed2a0bd140789f8bd699c8774675b22b18233d88a6421e +size 875569 diff --git a/m9j39xya/eval_video/3_eval.mp4 b/m9j39xya/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9a0c720c2fd238ff08e63b8043e0172f48335fd --- /dev/null +++ b/m9j39xya/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8dea06ae5d95c512f996672de99163d5a8c8a515c7b3738a0f8c9144dcb66d +size 1504374 diff --git a/m9j39xya/eval_video/4_eval.mp4 b/m9j39xya/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d109067684a6b27c2d8293662f389140dbe51ac7 --- /dev/null +++ b/m9j39xya/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6a2707d2dc8ca0cc1cdf4c42d4cd94ff7a445893aaaf91366584b3debadb70 +size 775015 diff --git a/m9j39xya/tb/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 b/m9j39xya/tb/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 new file mode 100644 index 0000000000000000000000000000000000000000..ae547b64478fc410e1f19f19c2bc0b5559e8884d --- /dev/null +++ b/m9j39xya/tb/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:187feb689ec17a6d870759e746dca69fad116dbf592f314ea2ac8241b029bb4a +size 6499 diff --git a/m9j39xya/wandb/debug-internal.log b/m9j39xya/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e43754124dfd26d35c57df63e1b32ff4fedf656d --- /dev/null +++ b/m9j39xya/wandb/debug-internal.log @@ -0,0 +1,681 @@ +2025-01-21 22:12:48,380 INFO StreamThr :2662652 [internal.py:wandb_internal():85] W&B internal server running at pid: 2662652, started at: 2025-01-21 22:12:48.380109 +2025-01-21 22:12:48,381 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:12:48,391 INFO WriterThread:2662652 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb +2025-01-21 22:12:48,391 DEBUG SenderThread:2662652 [sender.py:send():379] send: header +2025-01-21 22:12:48,399 DEBUG SenderThread:2662652 [sender.py:send():379] send: run +2025-01-21 22:12:48,650 INFO SenderThread:2662652 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files +2025-01-21 22:12:48,651 INFO SenderThread:2662652 [sender.py:_start_run_threads():1188] run started: m9j39xya with start time 1737515568.379828 +2025-01-21 22:12:48,666 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:12:48,666 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:12:48,712 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:12:48,718 DEBUG HandlerThread:2662652 [system_info.py:__init__():26] System info init +2025-01-21 22:12:48,718 DEBUG HandlerThread:2662652 [system_info.py:__init__():41] System info init done +2025-01-21 22:12:48,718 INFO HandlerThread:2662652 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:12:48,718 INFO SystemMonitor:2662652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:12:48,719 INFO HandlerThread:2662652 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:12:48,719 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:12:48,720 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:12:48,720 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:12:48,721 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:12:48,721 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:12:48,731 DEBUG HandlerThread:2662652 [system_info.py:probe():152] Probing system +2025-01-21 22:12:48,733 DEBUG HandlerThread:2662652 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:probe():200] Probing system done +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:12:48.731960', 'startedAt': '2025-01-22T03:12:48.371792', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.8822499999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.082, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.658, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 458.9532241821289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:12:48,738 INFO HandlerThread:2662652 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:12:48,738 INFO HandlerThread:2662652 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:_save_code():44] Saving code +2025-01-21 22:12:48,740 DEBUG HandlerThread:2662652 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:12:48,740 DEBUG HandlerThread:2662652 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:12:48,802 DEBUG HandlerThread:2662652 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:12:48,803 INFO HandlerThread:2662652 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:12:48,806 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:12:48,806 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:12:48,807 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:12:48,807 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:12:49,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:12:49,044 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:12:49,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:12:49,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:12:49,047 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:49,048 INFO HandlerThread:2662652 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/tb" + save: true +} +control { + relay_id: "e71301058cd64818874ab2ca48bb2810" +} +_info { + stream_id: "m9j39xya" +} + +2025-01-21 22:12:49,048 DEBUG HandlerThread:2662652 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:12:49,082 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/cxu1i13q-wandb-metadata.json +2025-01-21 22:12:49,117 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/bjqxhjsd-diff.patch +2025-01-21 22:12:49,128 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/8zwjhg9e-code/FISH/eval_robot.py +2025-01-21 22:12:49,145 DEBUG SenderThread:2662652 [sender.py:send():379] send: telemetry +2025-01-21 22:12:49,145 DEBUG SenderThread:2662652 [sender.py:send():379] send: tbrecord +2025-01-21 22:12:49,146 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:12:49,146 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 with policy live +2025-01-21 22:12:49,355 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/9qhq2t4e-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH +2025-01-21 22:12:50,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:51,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:51,651 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:52,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:53,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:54,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:12:54,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:54,886 DEBUG SenderThread:2662652 [sender.py:send():379] send: config +2025-01-21 22:12:55,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:56,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:57,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:57,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:58,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:59,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:12:59,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:59,654 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:00,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:01,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:01,655 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:02,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:03,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:03,655 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:04,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:04,044 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:04,085 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:04,120 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:05,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:06,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:07,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:08,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:09,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:10,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:10,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:11,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:12,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:13,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:14,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:15,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:15,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:15,658 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:16,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:17,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:17,659 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:18,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:19,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:19,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:19,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:19,660 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:20,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:20,203 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:20,660 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:13:21,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:22,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:23,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:24,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:25,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:26,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:26,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:27,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:28,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:29,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:30,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:31,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:31,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:32,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:33,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:33,305 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:13:33,306 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 with policy now +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:13:33,307 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:13:33,561 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/07z8jfo4-media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:33,663 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:33,663 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media +2025-01-21 22:13:33,933 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/v8ejxrtd-media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:34,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:34,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:34,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:35,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:35,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:36,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:36,160 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:37,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:38,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:39,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:39,665 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:40,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:40,666 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:13:41,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:41,246 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:41,666 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:42,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:43,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:43,667 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:44,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:45,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:45,667 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:46,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:47,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:47,197 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:48,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:48,721 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:13:48,722 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:13:49,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:49,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:49,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:50,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:51,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:52,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:52,425 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:53,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:53,670 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:54,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:55,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:56,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:57,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:58,050 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:58,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:59,069 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:59,278 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:13:59,279 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 with policy now +2025-01-21 22:13:59,279 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:13:59,280 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:13:59,280 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:13:59,282 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:13:59,539 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/q9hy6sqv-media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:13:59,950 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/321i8nyj-media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:14:00,071 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:01,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:01,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:02,080 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:03,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:03,673 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:03,761 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:04,048 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:04,048 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:04,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:05,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:05,674 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:06,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:07,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:07,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:08,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:08,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:08,940 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/c1jdov3y-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:09,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:09,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:09,426 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:09,426 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:09,426 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:09,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:09,676 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:10,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:11,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:11,676 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:12,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:13,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:13,677 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:14,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:14,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:15,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:15,677 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:16,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:17,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:18,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:18,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:14:19,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:19,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:19,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:19,121 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:20,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:21,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:22,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:23,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:24,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:25,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:25,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:26,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:27,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:28,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:29,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:30,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:30,325 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:31,080 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:31,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:31,837 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:14:31,837 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 with policy now +2025-01-21 22:14:31,842 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:14:31,842 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:31,842 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:31,843 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:32,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:32,122 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/0rvvzmki-media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:14:32,990 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/g38nfofy-media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:33,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:33,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:34,051 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:34,051 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:34,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:35,087 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:35,329 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:36,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:37,087 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:37,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:38,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:38,235 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:38,235 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:38,235 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:38,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:38,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:39,097 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:39,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:40,096 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:41,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:41,217 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:41,686 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:42,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:43,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:43,687 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:44,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:45,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:46,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:47,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:47,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:48,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:48,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:14:49,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:49,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:49,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:49,689 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:50,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:51,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:52,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:52,149 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:53,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:54,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:55,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:56,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:57,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:57,700 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:58,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:59,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:59,693 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:00,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:01,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:02,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:03,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:03,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:04,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:04,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:04,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:05,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:06,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:07,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:08,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:08,211 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:09,096 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:10,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:11,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:12,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:13,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:14,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:14,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:15,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:16,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:17,103 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:17,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:17,919 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:17,919 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 with policy now +2025-01-21 22:15:17,926 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:17,927 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:17,927 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:17,928 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:18,105 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:18,224 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/haftoyv5-media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:15:18,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:15:19,021 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/ka07kxd3-media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:19,056 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:19,056 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:19,118 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:19,179 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:19,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:20,131 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:21,117 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:22,122 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:23,123 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:23,701 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:24,119 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:24,426 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:25,123 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:25,701 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:26,121 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:27,120 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:28,128 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:28,702 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:28,988 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/giwzkjyn-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:29,126 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:29,296 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:29,296 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:29,297 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:29,485 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:29,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:29,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:30,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:31,140 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:31,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:32,139 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:33,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:33,704 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:34,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:34,047 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:34,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:35,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:35,184 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:36,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:37,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:38,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:39,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:40,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:41,067 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:41,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:42,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:43,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:44,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:45,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:46,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:47,067 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:47,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:48,138 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:48,724 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:15:49,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:49,059 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:49,143 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:49,269 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:49,269 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:49,269 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 with policy now +2025-01-21 22:15:49,277 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:49,277 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:49,278 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:49,531 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/0xzk0w1z-media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:15:49,992 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/q8ynkxpg-media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:50,144 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:51,156 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:51,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:52,157 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:52,311 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:53,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:54,159 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:55,154 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:55,712 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:56,159 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:57,158 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:57,330 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:57,330 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:57,331 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:57,331 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:57,712 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:57,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:57,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:57,825 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:15:57,825 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:15:58,055 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:58,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:58,366 INFO wandb-upload_1:2662652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzn7zvymn +2025-01-21 22:15:58,460 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:15:58,469 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table +2025-01-21 22:15:58,776 INFO SenderThread:2662652 [sender.py:send_request_log_artifact():1525] logged artifact run-m9j39xya-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyNjcxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNjAxOQ==', 'latestArtifact': None}} +2025-01-21 22:15:58,776 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:58,776 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 22:15:58,776 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,777 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send():379] send: telemetry +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,777 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,778 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,778 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,778 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,778 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,779 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,779 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,780 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,780 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,780 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,780 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,781 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,781 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,781 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,781 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,782 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,782 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,782 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,782 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:15:58,974 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/l7ae7t4o-media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:15:59,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:59,161 INFO wandb-upload_1:2662652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyded87vs +2025-01-21 22:15:59,172 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/z4u4eyki-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:15:59,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:59,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:59,738 INFO SenderThread:2662652 [sender.py:send_request_log_artifact():1525] logged artifact run-m9j39xya-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyNjczMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNjAyMA==', 'latestArtifact': None}} +2025-01-21 22:15:59,738 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:59,738 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json with policy now +2025-01-21 22:15:59,739 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:59,739 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:59,739 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:00,000 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/9rxyedcy-media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:16:00,002 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/aesz0j4c-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:16:00,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:00,714 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:01,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:02,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:02,743 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:03,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:03,715 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:16:04,048 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:04,048 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:04,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:05,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:06,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:07,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:08,147 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:08,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:09,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:10,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:11,154 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:11,474 DEBUG SenderThread:2662652 [sender.py:send():379] send: exit +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():588] handling runtime: 202 +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():594] send defer +2025-01-21 22:16:11,474 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,474 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:16:11,475 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:16:11,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:16:11,475 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:16:11,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:16:11,475 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:16:11,475 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:16:11,476 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:16:11,476 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:16:11,528 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:16:11,528 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:16:11,529 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:16:11,529 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,529 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:16:11,529 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:16:11,529 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:16:11,529 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,529 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:16:11,530 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,530 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:16:11,530 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:16:11,530 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,530 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:16:11,718 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:13,719 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,342 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:16:18,342 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:16:18,343 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:16:18,343 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,343 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:16:18,343 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:18,344 DEBUG SenderThread:2662652 [sender.py:send():379] send: summary +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:18,344 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:16:18,344 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,344 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:16:18,345 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,345 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:16:18,345 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:16:18,345 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:18,345 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,345 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:16:18,346 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,346 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:16:18,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:18,721 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:21,317 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:16:21,317 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:21,317 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,318 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:16:21,318 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,318 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:16:21,318 INFO SenderThread:2662652 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:16:21,320 INFO SenderThread:2662652 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:16:21,322 INFO SenderThread:2662652 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:16:21,335 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:16:21,336 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,336 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:16:21,336 DEBUG SenderThread:2662652 [sender.py:send():379] send: artifact +2025-01-21 22:16:21,476 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:21,669 INFO SenderThread:2662652 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:16:21,669 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,669 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:16:21,670 INFO SenderThread:2662652 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml config.yaml +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json wandb-summary.json +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt requirements.txt +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log output.log +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch diff.patch +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:16:21,730 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:21,731 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,732 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:16:21,732 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,733 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:16:21,733 INFO SenderThread:2662652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:16:21,919 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:21,930 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:16:21,951 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt +2025-01-21 22:16:21,966 INFO wandb-upload_4:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:22,004 INFO wandb-upload_3:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/5keozihf-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:16:22,205 INFO Thread-11 (_thread_body):2662652 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:16:22,205 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,205 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:16:22,205 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,205 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:16:22,205 INFO SenderThread:2662652 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:16:22,207 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,207 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:16:22,207 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:16:22,477 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,690 INFO SenderThread:2662652 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:16:22,690 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:16:22,690 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,690 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,691 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:16:22,691 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,691 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send():379] send: final +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send():379] send: footer +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:16:22,692 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,693 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:16:22,693 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:16:22,694 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:22,747 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,747 INFO MainThread:2662652 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:16:22,747 INFO MainThread:2662652 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:16:22,748 INFO MainThread:2662652 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:16:22,748 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:16:22,748 INFO HandlerThread:2662652 [handler.py:finish():882] shutting down handler +2025-01-21 22:16:23,692 INFO WriterThread:2662652 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [sender.py:finish():1615] shutting down sender +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [file_pusher.py:join():175] waiting for file pusher diff --git a/m9j39xya/wandb/debug.log b/m9j39xya/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8950ef2a3625db452e75e56044a0f27dac1d22c9 --- /dev/null +++ b/m9j39xya/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Configure stats pid to 2662476 +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/settings +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug.log +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug-internal.log +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya'} +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():619] starting backend +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():623] setting up manager +2025-01-21 22:12:48,379 INFO MainThread:2662476 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:12:48,379 INFO MainThread:2662476 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:12:48,391 INFO MainThread:2662476 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:12:48,397 INFO MainThread:2662476 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:12:48,666 INFO MainThread:2662476 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:12:48,706 INFO MainThread:2662476 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:12:48,706 INFO MainThread:2662476 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:12:49,045 INFO MainThread:2662476 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:12:49,047 INFO MainThread:2662476 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:12:49,047 INFO MainThread:2662476 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/tb, True +2025-01-21 22:12:54,885 INFO MainThread:2662476 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:16:23,748 WARNING MsgRouterThr:2662476 [router.py:message_loop():77] message_loop has been closed diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22f60e87582fa993c56e4bcb37130c54d3ab2833 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737515568 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ad86d1c16f9f01552bf32121810c12f23e78dac5 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[12, 40, 0.3, 0.1515692175131524, 0.48537603020960507, 0.17795732623819088, 0.46414706111671356, 0.19561152166255463, 0.4242345318349736, 0.20827673753970033, 0.40881603212988127]]} \ No newline at end of file diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0c60d96f1d5d1a5bf6422eeacb5dc361cca7e91 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be41a6a3e5ef05ec42a0b86fd98317313bee30f6316ecc0cfb731364ee60638d +size 726574 diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff6e6c31a2343624d224a56c6539ba1074fa98b8 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad3604e3344632866d9dde11b11155716453c35ab19983ca5cc1ac428f91b64b +size 704030 diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..347581392d8131a045fae763c6a09108cc99d25b --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11990a56a7de0e8d16ed2a0bd140789f8bd699c8774675b22b18233d88a6421e +size 875569 diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9a0c720c2fd238ff08e63b8043e0172f48335fd --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8dea06ae5d95c512f996672de99163d5a8c8a515c7b3738a0f8c9144dcb66d +size 1504374 diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d109067684a6b27c2d8293662f389140dbe51ac7 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6a2707d2dc8ca0cc1cdf4c42d4cd94ff7a445893aaaf91366584b3debadb70 +size 775015 diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9953c56001793a4220052edd130895701423ec11 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log @@ -0,0 +1,295 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737515574.886862]: resetting environment +[INFO] [1737515574.891460]: cleared current plan +[INFO] [1737515574.892372]: moving to home +[INFO] [1737515575.894599]: reached home +[INFO] [1737515575.896251]: reset action history +[INFO] [1737515578.609892]: environment reset +Starting episode 0 +[INFO] [1737515578.614101]: resetting environment +[INFO] [1737515578.618960]: cleared current plan +[INFO] [1737515578.619889]: moving to home +[INFO] [1737515579.621266]: reached home +[INFO] [1737515579.622846]: reset action history +[INFO] [1737515582.332425]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737515594.085824]: resetting environment +[INFO] [1737515594.091626]: cleared current plan +[INFO] [1737515594.092489]: moving to home +[INFO] [1737515595.095011]: reached home +[INFO] [1737515595.096413]: reset action history +[INFO] [1737515597.805964]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/episode_rosbags/episode_0_2025-01-21-22-13-17.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2662476.log +started roslaunch server http://158.130.50.37:40233/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2662782] +process[print_text-2]: started with pid [2662783] +process[rosbag_record-3]: started with pid [2662784] +started bagging! +preempting episode +[INFO] [1737515612.299610]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737515613.044094]: Stored episode 1. +[INFO] [1737515613.044316]: Saving video... +[INFO] [1737515613.258103]: Video saved! +Starting episode 1 +[INFO] [1737515613.268602]: resetting environment +[INFO] [1737515613.281744]: cleared current plan +[INFO] [1737515613.282673]: moving to home +[INFO] [1737515616.785276]: reached home +[INFO] [1737515616.785746]: reset action history +[INFO] [1737515619.495389]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515620.233352]: resetting environment +[INFO] [1737515620.242709]: cleared current plan +[INFO] [1737515620.243452]: moving to home +[INFO] [1737515621.245321]: reached home +[INFO] [1737515621.246495]: reset action history +[INFO] [1737515623.955157]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/episode_rosbags/episode_1_2025-01-21-22-13-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2662476.log +started roslaunch server http://158.130.50.37:37473/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2662912] +process[print_text-5]: started with pid [2662913] +process[rosbag_record-6]: started with pid [2662914] +started bagging! +For topic gripper_width: timestamp difference is 100855982 for nearest: 1737515631379169857 - target: 1737515631278313875 at idx 111 +preempting episode +[INFO] [1737515638.072051]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[WARN] [1737515639.185873]: Plan exhausted +[WARN] [1737515639.225284]: Plan exhausted +[WARN] [1737515639.264760]: Plan exhausted +[INFO] [1737515638.866535]: Stored episode 2. +[INFO] [1737515638.866794]: Saving video... +[INFO] [1737515639.243041]: Video saved! +Starting episode 2 +[INFO] [1737515639.282781]: resetting environment +[INFO] [1737515639.302275]: cleared current plan +[INFO] [1737515639.309244]: moving to home +[INFO] [1737515642.717269]: reached home +[INFO] [1737515642.719943]: reset action history +[INFO] [1737515645.517742]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515648.387467]: resetting environment +[INFO] [1737515648.391995]: cleared current plan +[INFO] [1737515648.392328]: moving to home +[INFO] [1737515649.393318]: reached home +[INFO] [1737515649.393779]: reset action history +[INFO] [1737515652.104237]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/episode_rosbags/episode_2_2025-01-21-22-14-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2662476.log +started roslaunch server http://158.130.50.37:37541/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2663088] +process[print_text-8]: started with pid [2663089] +process[rosbag_record-9]: started with pid [2663090] +started bagging! +For topic gripper_width: timestamp difference is -108068997 for nearest: 1737515653112461751 - target: 1737515653220530748 at idx 14 +preempting episode +[INFO] [1737515670.325459]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[WARN] [1737515671.439079]: Plan exhausted +[WARN] [1737515671.479612]: Plan exhausted +[WARN] [1737515671.523135]: Plan exhausted +[WARN] [1737515671.558892]: Plan exhausted +[WARN] [1737515671.599076]: Plan exhausted +[WARN] [1737515671.642916]: Plan exhausted +[WARN] [1737515671.679866]: Plan exhausted +[WARN] [1737515671.718886]: Plan exhausted +[WARN] [1737515671.759292]: Plan exhausted +[WARN] [1737515671.802116]: Plan exhausted +[WARN] [1737515671.839838]: Plan exhausted +[INFO] [1737515671.081963]: Stored episode 3. +[INFO] [1737515671.082631]: Saving video... +[INFO] [1737515671.790240]: Video saved! +Starting episode 3 +[INFO] [1737515671.845620]: resetting environment +[INFO] [1737515671.875659]: cleared current plan +[INFO] [1737515671.876257]: moving to home +[INFO] [1737515675.281082]: reached home +[INFO] [1737515675.292200]: reset action history +[INFO] [1737515678.096626]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515678.169655]: resetting environment +[INFO] [1737515678.173449]: cleared current plan +[INFO] [1737515678.173659]: moving to home +[INFO] [1737515679.175266]: reached home +[INFO] [1737515679.179466]: reset action history +[INFO] [1737515682.020619]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/episode_rosbags/episode_3_2025-01-21-22-14-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2662476.log +started roslaunch server http://158.130.50.37:41265/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2663204] +process[print_text-11]: started with pid [2663205] +process[rosbag_record-12]: started with pid [2663206] +started bagging! +For topic gripper_width: timestamp difference is -152175599 for nearest: 1737515687912468499 - target: 1737515688064644098 at idx 77 +For topic gripper_width: timestamp difference is -110822969 for nearest: 1737515697512461132 - target: 1737515697623284101 at idx 198 +For topic gripper_width: timestamp difference is 109592179 for nearest: 1737515716412460545 - target: 1737515716302868366 at idx 249 +[INFO] [1737515716.651694]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1737515717.409181]: Stored episode 4. +[INFO] [1737515717.410858]: Saving video... +[INFO] [1737515717.863780]: Video saved! +Starting episode 4 +[INFO] [1737515717.934672]: resetting environment +[INFO] [1737515717.990645]: cleared current plan +[INFO] [1737515717.992977]: moving to home +[INFO] [1737515721.613789]: reached home +[INFO] [1737515721.626184]: reset action history +[INFO] [1737515724.422722]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515728.424570]: resetting environment +[INFO] [1737515728.459790]: cleared current plan +[INFO] [1737515728.470942]: moving to home +[INFO] [1737515729.482886]: reached home +[INFO] [1737515729.487236]: reset action history +[INFO] [1737515732.284697]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/episode_rosbags/episode_4_2025-01-21-22-15-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2662476.log +started roslaunch server http://158.130.50.37:45735/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2663367] +process[print_text-14]: started with pid [2663368] +process[rosbag_record-15]: started with pid [2663369] +started bagging! +preempting episode +[INFO] [1737515747.748940]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737515748.642887]: Stored episode 5. +[INFO] [1737515748.645425]: Saving video... +[WARN] [1737515748.919814]: Plan exhausted +[WARN] [1737515748.961289]: Plan exhausted +[WARN] [1737515749.000411]: Plan exhausted +[WARN] [1737515749.041126]: Plan exhausted +[WARN] [1737515749.079604]: Plan exhausted +[WARN] [1737515749.119135]: Plan exhausted +[WARN] [1737515749.158875]: Plan exhausted +[WARN] [1737515749.201201]: Plan exhausted +[WARN] [1737515749.240967]: Plan exhausted +[WARN] [1737515749.279006]: Plan exhausted +[INFO] [1737515749.225152]: Video saved! +[INFO] [1737515749.270144]: resetting environment +[INFO] [1737515749.306711]: cleared current plan +[INFO] [1737515749.309036]: moving to home +[INFO] [1737515752.916498]: reached home +[INFO] [1737515752.925918]: reset action history +[INFO] [1737515755.762858]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bb96338fe1a0ed254db589dbe28e9a5bdda4170f --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T03:12:48.731960", + "startedAt": "2025-01-22T03:12:48.371792", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.8822499999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.991, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.082, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.658, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.024, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 458.9532241821289 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f149e29022edaa8fad41fe908535624287a57e17 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "be41a6a3e5ef05ec42a0b86fd98317313bee30f6316ecc0cfb731364ee60638d", "size": 726574, "path": "media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4"}, "global_step": 365, "_timestamp": 1737515758.3607316, "_runtime": 189.98090362548828, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ad3604e3344632866d9dde11b11155716453c35ab19983ca5cc1ac428f91b64b", "size": 704030, "path": "media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4"}, "eval/num_success": 1.0, "episode": 40.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "11990a56a7de0e8d16ed2a0bd140789f8bd699c8774675b22b18233d88a6421e", "size": 875569, "path": "media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9c8dea06ae5d95c512f996672de99163d5a8c8a515c7b3738a0f8c9144dcb66d", "size": 1504374, "path": "media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ea6a2707d2dc8ca0cc1cdf4c42d4cd94ff7a445893aaaf91366584b3debadb70", "size": 775015, "path": "media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://3bghrvax3so7un71k7cdfcu93rcitd55sgr58bi0rrwi1ykagihwwug04v8wyy5p3xikg5ui2ed2mg1fjfshwt6yz681f04xwtbvxz02v76vkh9r8s6kkkmfppt2hmut/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ohcd2jgq5i8l5m145ouszzzmmpm9i3j84wwn95qyh3n92oe94enuz6kj9v0syziv7vplisguzpx4yp29prcv6a99jh641donmsu539i0619er6t8ckcn7ycdab7qhcf5:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 12, "total_num_episodes": 40, "total_success_rate": 0.3, "total_success_rate_lb_0.01": 0.1515692175131524, "total_success_rate_ub_0.01": 0.48537603020960507, "total_success_rate_lb_0.025": 0.17795732623819088, "total_success_rate_ub_0.025": 0.46414706111671356, "total_success_rate_lb_0.05": 0.19561152166255463, "total_success_rate_ub_0.05": 0.4242345318349736, "total_success_rate_lb_0.1": 0.20827673753970033, "total_success_rate_ub_0.1": 0.40881603212988127, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "d5c87daf8c809da084e2d13d380047162a18de4438a9ffc8f0fb93af56cc26ab", "size": 511, "artifact_path": "wandb-client-artifact://wjs9qij3m90ceph8i7363po6zm5s6olk9936kt9ozb1gb7xu8u6x1x00vg61uh3zmr5dj9qinv2vlbg8mkdgku63f7227i5se167gvmwnvpcu2i01j018a5au7r168hk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://l89ctq6wzkwi7nddul8rbqzp18af6tnhqo30ob7olbqfen9x0ir0yhgdpjhs1lk34ptxb0ffu8n7suc5c1tmry2kx18yl4rrvm2faloqvh07kl8wtgdxpb5ec4lcnkle:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 202}, "eval/total_success_rate": 0.30000001192092896, "eval/total_success_rate_lb_0.01": 0.1515692174434662, "eval/total_success_rate_ub_0.01": 0.4853760302066803, "eval/total_success_rate_lb_0.025": 0.17795732617378235, "eval/total_success_rate_ub_0.025": 0.46414706110954285, "eval/total_success_rate_lb_0.05": 0.19561152160167694, "eval/total_success_rate_ub_0.05": 0.424234539270401, "eval/total_success_rate_lb_0.1": 0.20827673375606537, "eval/total_success_rate_ub_0.1": 0.40881603956222534} \ No newline at end of file diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug-internal.log b/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e43754124dfd26d35c57df63e1b32ff4fedf656d --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug-internal.log @@ -0,0 +1,681 @@ +2025-01-21 22:12:48,380 INFO StreamThr :2662652 [internal.py:wandb_internal():85] W&B internal server running at pid: 2662652, started at: 2025-01-21 22:12:48.380109 +2025-01-21 22:12:48,381 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:12:48,391 INFO WriterThread:2662652 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb +2025-01-21 22:12:48,391 DEBUG SenderThread:2662652 [sender.py:send():379] send: header +2025-01-21 22:12:48,399 DEBUG SenderThread:2662652 [sender.py:send():379] send: run +2025-01-21 22:12:48,650 INFO SenderThread:2662652 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files +2025-01-21 22:12:48,651 INFO SenderThread:2662652 [sender.py:_start_run_threads():1188] run started: m9j39xya with start time 1737515568.379828 +2025-01-21 22:12:48,666 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:12:48,666 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:12:48,712 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:12:48,718 DEBUG HandlerThread:2662652 [system_info.py:__init__():26] System info init +2025-01-21 22:12:48,718 DEBUG HandlerThread:2662652 [system_info.py:__init__():41] System info init done +2025-01-21 22:12:48,718 INFO HandlerThread:2662652 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:12:48,718 INFO SystemMonitor:2662652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:12:48,719 INFO HandlerThread:2662652 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:12:48,719 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:12:48,720 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:12:48,720 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:12:48,721 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:12:48,721 INFO SystemMonitor:2662652 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:12:48,731 DEBUG HandlerThread:2662652 [system_info.py:probe():152] Probing system +2025-01-21 22:12:48,733 DEBUG HandlerThread:2662652 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:probe():200] Probing system done +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:12:48.731960', 'startedAt': '2025-01-22T03:12:48.371792', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.8822499999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.082, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.658, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 458.9532241821289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:12:48,738 INFO HandlerThread:2662652 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:12:48,738 INFO HandlerThread:2662652 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:12:48,738 DEBUG HandlerThread:2662652 [system_info.py:_save_code():44] Saving code +2025-01-21 22:12:48,740 DEBUG HandlerThread:2662652 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:12:48,740 DEBUG HandlerThread:2662652 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:12:48,802 DEBUG HandlerThread:2662652 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:12:48,803 INFO HandlerThread:2662652 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:12:48,806 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:12:48,806 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:12:48,807 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:12:48,807 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:12:49,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:12:49,044 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:12:49,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:12:49,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:12:49,047 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:49,048 INFO HandlerThread:2662652 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/tb" + save: true +} +control { + relay_id: "e71301058cd64818874ab2ca48bb2810" +} +_info { + stream_id: "m9j39xya" +} + +2025-01-21 22:12:49,048 DEBUG HandlerThread:2662652 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:12:49,082 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/cxu1i13q-wandb-metadata.json +2025-01-21 22:12:49,117 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/bjqxhjsd-diff.patch +2025-01-21 22:12:49,128 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/8zwjhg9e-code/FISH/eval_robot.py +2025-01-21 22:12:49,145 DEBUG SenderThread:2662652 [sender.py:send():379] send: telemetry +2025-01-21 22:12:49,145 DEBUG SenderThread:2662652 [sender.py:send():379] send: tbrecord +2025-01-21 22:12:49,146 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:12:49,146 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 with policy live +2025-01-21 22:12:49,355 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/9qhq2t4e-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json +2025-01-21 22:12:49,652 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code +2025-01-21 22:12:49,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH +2025-01-21 22:12:50,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:51,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:51,651 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:52,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:53,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:54,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:12:54,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:54,886 DEBUG SenderThread:2662652 [sender.py:send():379] send: config +2025-01-21 22:12:55,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:56,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:57,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:57,653 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:12:58,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:59,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:12:59,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:12:59,654 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:00,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:01,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:01,655 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:02,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:03,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:03,655 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:04,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:04,044 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:04,085 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:04,120 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:05,057 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:06,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:07,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:08,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:09,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:10,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:10,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:11,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:12,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:13,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:14,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:15,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:15,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:15,658 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:16,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:17,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:17,659 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:18,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:19,044 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:19,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:19,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:19,660 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:20,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:20,203 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:20,660 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:13:21,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:22,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:23,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:24,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:25,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:26,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:26,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:27,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:28,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:29,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:30,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:31,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:31,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:32,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:33,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:33,305 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:13:33,306 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 with policy now +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:13:33,306 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:13:33,307 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:13:33,561 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/07z8jfo4-media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:33,663 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:33,663 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:13:33,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media +2025-01-21 22:13:33,933 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/v8ejxrtd-media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:13:34,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:34,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:34,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:35,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:35,664 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:36,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:36,160 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:37,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:38,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:39,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:39,665 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:40,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:40,666 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:13:41,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:41,246 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:41,666 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:42,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:43,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:43,667 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:44,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:45,062 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:45,667 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:46,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:47,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:47,197 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:48,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:48,721 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:13:48,722 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:13:49,045 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:13:49,045 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:13:49,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:50,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:51,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:52,063 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:52,425 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:53,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:53,670 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:54,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:55,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:56,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:57,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:58,050 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:13:58,064 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:59,069 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:13:59,278 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:13:59,279 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 with policy now +2025-01-21 22:13:59,279 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:13:59,280 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:13:59,280 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:13:59,282 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:13:59,539 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/q9hy6sqv-media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:13:59,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:13:59,950 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/321i8nyj-media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:14:00,071 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:01,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:01,672 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:02,080 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:03,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:03,673 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:03,761 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:04,048 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:04,048 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:04,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:05,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:05,674 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:06,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:07,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:07,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:08,073 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:08,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:08,940 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/c1jdov3y-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:09,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:09,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:09,426 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:09,426 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:09,426 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:09,675 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:09,676 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:10,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:11,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:11,676 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:12,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:13,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:13,677 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:14,059 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:14,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:15,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:15,677 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:16,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:17,074 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:18,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:18,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:14:19,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:19,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:19,089 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:19,121 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:20,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:21,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:22,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:23,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:24,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:25,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:25,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:26,075 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:27,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:28,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:29,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:30,076 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:30,325 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:31,080 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:31,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:31,837 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:14:31,837 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 with policy now +2025-01-21 22:14:31,842 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:14:31,842 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:31,842 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:31,843 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:32,077 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:32,122 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/0rvvzmki-media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:32,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:14:32,990 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/g38nfofy-media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:14:33,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:33,683 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:34,051 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:34,051 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:34,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:35,087 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:35,329 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:36,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:37,087 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:37,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:38,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:38,235 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:14:38,235 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:14:38,235 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:14:38,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:14:38,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:14:39,097 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:39,685 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:40,096 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:41,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:41,217 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:41,686 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:42,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:43,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:43,687 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:44,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:45,090 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:46,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:47,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:47,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:48,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:48,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:14:49,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:14:49,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:14:49,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:49,689 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:14:50,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:51,091 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:52,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:52,149 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:53,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:54,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:55,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:56,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:57,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:57,700 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:14:58,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:59,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:14:59,693 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:00,092 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:01,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:02,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:03,060 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:03,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:04,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:04,046 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:04,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:05,093 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:06,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:07,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:08,094 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:08,211 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:09,096 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:10,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:11,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:12,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:13,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:14,061 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:14,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:15,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:16,095 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:17,103 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:17,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:17,919 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:17,919 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 with policy now +2025-01-21 22:15:17,926 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:17,927 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:17,927 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:17,928 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:18,105 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:18,224 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/haftoyv5-media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:18,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:15:18,723 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:15:19,021 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/ka07kxd3-media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:15:19,056 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:19,056 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:19,118 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:19,179 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:19,699 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:20,131 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:21,117 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:22,122 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:23,123 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:23,701 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:24,119 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:24,426 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:25,123 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:25,701 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:26,121 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:27,120 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:28,128 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:28,702 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:28,988 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/giwzkjyn-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:29,126 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:29,296 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:29,296 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:29,297 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:29,485 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:29,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:29,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:30,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:31,140 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:31,703 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:32,139 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:33,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:33,704 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:34,046 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:34,047 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:34,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:35,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:35,184 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:36,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:37,134 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:38,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:39,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:40,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:41,067 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:41,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:42,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:43,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:44,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:45,135 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:46,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:47,067 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:47,136 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:48,138 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:48,724 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:15:49,058 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:15:49,059 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:15:49,143 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:49,269 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:49,269 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:49,269 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 with policy now +2025-01-21 22:15:49,277 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:49,277 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:49,278 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:49,531 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/0xzk0w1z-media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:49,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval +2025-01-21 22:15:49,992 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/q8ynkxpg-media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:15:50,144 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:51,156 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:51,710 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:52,157 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:52,311 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:53,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:54,159 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:55,154 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:55,712 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:56,159 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:57,158 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:57,330 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:57,330 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:57,331 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:15:57,331 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:57,712 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:57,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:57,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:57,825 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:15:57,825 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:15:58,055 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:58,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:58,366 INFO wandb-upload_1:2662652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzn7zvymn +2025-01-21 22:15:58,460 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:15:58,469 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval +2025-01-21 22:15:58,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table +2025-01-21 22:15:58,776 INFO SenderThread:2662652 [sender.py:send_request_log_artifact():1525] logged artifact run-m9j39xya-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyNjcxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNjAxOQ==', 'latestArtifact': None}} +2025-01-21 22:15:58,776 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:58,776 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 22:15:58,776 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,777 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send():379] send: telemetry +2025-01-21 22:15:58,777 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,777 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,778 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,778 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,778 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,778 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,779 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,779 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,779 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,780 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,780 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,780 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,780 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,781 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,781 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,781 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,781 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,782 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,782 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:58,782 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:15:58,782 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:15:58,974 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/l7ae7t4o-media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:15:59,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:15:59,161 INFO wandb-upload_1:2662652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyded87vs +2025-01-21 22:15:59,172 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/z4u4eyki-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:15:59,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:15:59,713 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:15:59,738 INFO SenderThread:2662652 [sender.py:send_request_log_artifact():1525] logged artifact run-m9j39xya-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyNjczMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNjAyMA==', 'latestArtifact': None}} +2025-01-21 22:15:59,738 DEBUG SenderThread:2662652 [sender.py:send():379] send: files +2025-01-21 22:15:59,738 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json with policy now +2025-01-21 22:15:59,739 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:15:59,739 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:15:59,739 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:00,000 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/9rxyedcy-media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:16:00,002 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/aesz0j4c-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:16:00,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:00,714 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:01,152 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:02,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:02,743 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:03,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:03,715 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:16:04,048 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:04,048 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:04,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:05,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:06,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:07,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:08,147 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:08,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:09,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:10,153 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:11,154 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:11,474 DEBUG SenderThread:2662652 [sender.py:send():379] send: exit +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():588] handling runtime: 202 +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:11,474 INFO SenderThread:2662652 [sender.py:send_exit():594] send defer +2025-01-21 22:16:11,474 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,474 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:16:11,475 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:16:11,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:16:11,475 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:16:11,475 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:16:11,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:16:11,475 INFO HandlerThread:2662652 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:16:11,475 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:16:11,475 DEBUG SystemMonitor:2662652 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:16:11,476 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:16:11,476 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:16:11,528 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:16:11,528 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:16:11,529 INFO HandlerThread:2662652 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:16:11,529 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,529 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:16:11,529 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:16:11,529 DEBUG SenderThread:2662652 [sender.py:send():379] send: stats +2025-01-21 22:16:11,529 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,529 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:16:11,530 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:11,530 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:16:11,530 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:16:11,530 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:11,530 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:16:11,718 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:13,719 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:18,341 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,342 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:16:18,342 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:18,342 DEBUG SenderThread:2662652 [sender.py:send():379] send: history +2025-01-21 22:16:18,343 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:16:18,343 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,343 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:16:18,343 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:18,344 DEBUG SenderThread:2662652 [sender.py:send():379] send: summary +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:16:18,344 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:16:18,344 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:16:18,344 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,344 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:16:18,345 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,345 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:16:18,345 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:16:18,345 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:18,345 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:18,345 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:16:18,346 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:18,346 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:16:18,475 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:18,721 INFO Thread-12 :2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:21,317 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:16:21,317 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:21,317 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,318 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:16:21,318 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,318 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:16:21,318 INFO SenderThread:2662652 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:16:21,320 INFO SenderThread:2662652 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:16:21,322 INFO SenderThread:2662652 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:16:21,335 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:16:21,336 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,336 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:16:21,336 DEBUG SenderThread:2662652 [sender.py:send():379] send: artifact +2025-01-21 22:16:21,476 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:21,669 INFO SenderThread:2662652 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:16:21,669 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,669 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:16:21,670 INFO SenderThread:2662652 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files +2025-01-21 22:16:21,722 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml config.yaml +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json wandb-summary.json +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt requirements.txt +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:16:21,723 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log output.log +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/diff.patch diff.patch +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:16:21,726 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 media/videos/eval/3_eval_3_9c8dea06ae5d95c512f9.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 media/videos/eval/2_eval_2_11990a56a7de0e8d16ed.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 media/videos/eval/4_eval_4_ea6a2707d2dc8ca0cc1c.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 media/videos/eval/0_eval_0_be41a6a3e5ef05ec42a0.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 media/videos/eval/1_eval_1_ad3604e3344632866d9d.mp4 +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json media/table/eval/total_success_rate_ci_6_d5c87daf8c809da084e2.table.json +2025-01-21 22:16:21,727 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:16:21,730 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:21,731 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:21,732 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:16:21,732 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:21,733 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:16:21,733 INFO SenderThread:2662652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:16:21,919 INFO wandb-upload_2:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/wandb-summary.json +2025-01-21 22:16:21,930 INFO wandb-upload_1:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/config.yaml +2025-01-21 22:16:21,951 INFO wandb-upload_0:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/requirements.txt +2025-01-21 22:16:21,966 INFO wandb-upload_4:2662652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/files/output.log +2025-01-21 22:16:22,004 INFO wandb-upload_3:2662652 [upload_job.py:push():130] Uploaded file /tmp/tmpecnuv76uwandb/5keozihf-events.out.tfevents.1737515569.leonmkim-ROG-Strix-G15CS-G15CS.2662476.0 +2025-01-21 22:16:22,205 INFO Thread-11 (_thread_body):2662652 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:16:22,205 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,205 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:16:22,205 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,205 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:16:22,205 INFO SenderThread:2662652 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:16:22,207 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,207 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:16:22,207 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:16:22,207 INFO SenderThread:2662652 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:16:22,477 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,690 INFO SenderThread:2662652 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:16:22,690 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:16:22,690 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,690 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,691 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:16:22,691 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:16:22,691 INFO HandlerThread:2662652 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send():379] send: final +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send():379] send: footer +2025-01-21 22:16:22,691 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: defer +2025-01-21 22:16:22,691 INFO SenderThread:2662652 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:16:22,692 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:16:22,692 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,693 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:16:22,693 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:16:22,694 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:22,747 DEBUG SenderThread:2662652 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:16:22,747 INFO MainThread:2662652 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:16:22,747 INFO MainThread:2662652 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:16:22,748 INFO MainThread:2662652 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:16:22,748 DEBUG HandlerThread:2662652 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:16:22,748 INFO HandlerThread:2662652 [handler.py:finish():882] shutting down handler +2025-01-21 22:16:23,692 INFO WriterThread:2662652 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [sender.py:finish():1615] shutting down sender +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:16:23,747 INFO SenderThread:2662652 [file_pusher.py:join():175] waiting for file pusher diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug.log b/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8950ef2a3625db452e75e56044a0f27dac1d22c9 --- /dev/null +++ b/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Configure stats pid to 2662476 +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/settings +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug.log +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/wandb/run-20250121_221248-m9j39xya/logs/debug-internal.log +2025-01-21 22:12:48,374 INFO MainThread:2662476 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya'} +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():619] starting backend +2025-01-21 22:12:48,375 INFO MainThread:2662476 [wandb_init.py:init():623] setting up manager +2025-01-21 22:12:48,379 INFO MainThread:2662476 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:12:48,379 INFO MainThread:2662476 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:12:48,391 INFO MainThread:2662476 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:12:48,397 INFO MainThread:2662476 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:12:48,666 INFO MainThread:2662476 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:12:48,706 INFO MainThread:2662476 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:12:48,706 INFO MainThread:2662476 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:12:49,044 INFO MainThread:2662476 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:12:49,045 INFO MainThread:2662476 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:12:49,047 INFO MainThread:2662476 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:12:49,047 INFO MainThread:2662476 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/m9j39xya/tb, True +2025-01-21 22:12:54,885 INFO MainThread:2662476 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:16:23,748 WARNING MsgRouterThr:2662476 [router.py:message_loop():77] message_loop has been closed diff --git a/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb b/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f4f38bfe99130b94a83a6bf290e568c9cd406e2b Binary files /dev/null and b/m9j39xya/wandb/run-20250121_221248-m9j39xya/run-m9j39xya.wandb differ diff --git a/n3ccs0ug/.hydra/config.yaml b/n3ccs0ug/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/n3ccs0ug/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/n3ccs0ug/.hydra/hydra.yaml b/n3ccs0ug/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bafdc4958560105e4e2a44cb2660ffeb0528264 --- /dev/null +++ b/n3ccs0ug/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/n3ccs0ug + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/n3ccs0ug/.hydra/overrides.yaml b/n3ccs0ug/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/n3ccs0ug/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/n3ccs0ug/eval_policy.log b/n3ccs0ug/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..fee966c9b5b14c6e9340c73d6f6f8b9f1552cc2b --- /dev/null +++ b/n3ccs0ug/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 20:55:58,935][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:55:58,941][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 20:56:02,705][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ad8dc88c496e7b99ff08b8f86bd8f3f19ed1626 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ec5c613e37a38b28314cc2a095c120aaf76c2a750a2944119b162c97bc47a2 +size 910332870 diff --git a/v3cy5mau/.hydra/config.yaml b/v3cy5mau/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/v3cy5mau/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/v3cy5mau/.hydra/hydra.yaml b/v3cy5mau/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..645293e93f6dabab0a14923d3bc9852a6617dcce --- /dev/null +++ b/v3cy5mau/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/v3cy5mau + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/v3cy5mau/.hydra/overrides.yaml b/v3cy5mau/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/v3cy5mau/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/v3cy5mau/eval_policy.log b/v3cy5mau/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..39eaa615a066f40f958947ec40b4ffaad5c1e716 --- /dev/null +++ b/v3cy5mau/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 21:35:42,150][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:35:42,153][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 21:35:46,481][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +