diff --git a/.gitattributes b/.gitattributes index 1b676544edb45c48a9fc8a1566ed8a4accc6fc70..af582eea601ef159cb2750276471cfff81162e72 100644 --- a/.gitattributes +++ b/.gitattributes @@ -92,3 +92,60 @@ uh3y27hm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text uh3y27hm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text uh3y27hm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text uh3y27hm/episode_rosbags/episode_1_2025-01-01-13-30-43.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/episode_rosbags/episode_3_2025-01-01-13-54-52.bag filter=lfs diff=lfs merge=lfs -text +niwmozqn/episode_rosbags/episode_2_2025-01-01-13-53-54.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/episode_rosbags/episode_1_2025-01-01-13-52-57.bag filter=lfs diff=lfs merge=lfs -text +71icjjya/episode_rosbags/episode_0_2024-12-31-23-53-29.bag filter=lfs diff=lfs merge=lfs -text +71icjjya/episode_rosbags/episode_1_2024-12-31-23-54-45.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/episode_rosbags/episode_0_2024-12-31-23-38-10.bag filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/episode_rosbags/episode_0_2025-01-01-13-52-01.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/episode_rosbags/episode_2_2024-12-31-23-39-50.bag filter=lfs diff=lfs merge=lfs -text +71icjjya/episode_rosbags/episode_3_2024-12-31-23-56-50.bag filter=lfs diff=lfs merge=lfs -text +niwmozqn/episode_rosbags/episode_4_2025-01-01-13-55-53.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/episode_rosbags/episode_4_2024-12-31-23-41-30.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/episode_rosbags/episode_1_2024-12-31-23-38-59.bag filter=lfs diff=lfs merge=lfs -text +h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/episode_rosbags/episode_4_2024-12-31-23-58-00.bag filter=lfs diff=lfs merge=lfs -text +h2q92ngg/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h2q92ngg/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/episode_rosbags/episode_4_2024-12-31-22-37-46.bag filter=lfs diff=lfs merge=lfs -text +dr7dcnpy/episode_rosbags/episode_3_2024-12-31-23-40-40.bag filter=lfs diff=lfs merge=lfs -text +3bbvaqak/episode_rosbags/episode_3_2024-12-31-22-36-52.bag filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 filter=lfs diff=lfs merge=lfs -text +kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7kvn4h3l/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3bbvaqak/episode_rosbags/episode_3_2024-12-31-22-36-52.bag b/3bbvaqak/episode_rosbags/episode_3_2024-12-31-22-36-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..4d3228f308c40a700dde2112f904d918d88bd5f1 --- /dev/null +++ b/3bbvaqak/episode_rosbags/episode_3_2024-12-31-22-36-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ec1e040149308ad948d30af8f804e1b07ab0cb16751b43d7f4a6f82d480b67 +size 2608774920 diff --git a/3bbvaqak/episode_rosbags/episode_4_2024-12-31-22-37-46.bag b/3bbvaqak/episode_rosbags/episode_4_2024-12-31-22-37-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..5cd635a2afa538820e03b9775f23370bf83af6ae --- /dev/null +++ b/3bbvaqak/episode_rosbags/episode_4_2024-12-31-22-37-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3721ace4fb7d13a123ddecb004efe15f56b4eae4d60f236d00e0a0825db0884b +size 2610136192 diff --git a/71icjjya/episode_rosbags/episode_0_2024-12-31-23-53-29.bag b/71icjjya/episode_rosbags/episode_0_2024-12-31-23-53-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..d71cbcf34fd2c5295f0dc35aa13a30408abec9e3 --- /dev/null +++ b/71icjjya/episode_rosbags/episode_0_2024-12-31-23-53-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:613dce8f47f8771fb60a1c12a43242b34b60570e6d19c1536a6feafc94aa333b +size 2601849578 diff --git a/71icjjya/episode_rosbags/episode_1_2024-12-31-23-54-45.bag b/71icjjya/episode_rosbags/episode_1_2024-12-31-23-54-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..8cd4a9a03f3b3268349306e9b295b4f9ee832178 --- /dev/null +++ b/71icjjya/episode_rosbags/episode_1_2024-12-31-23-54-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9cef567a5d379f334b1ad40aef2a87769ae954d644c6d079f21b3072d342677 +size 2507087764 diff --git a/71icjjya/episode_rosbags/episode_3_2024-12-31-23-56-50.bag b/71icjjya/episode_rosbags/episode_3_2024-12-31-23-56-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..c1166c002e1231327fd56aa22d0a0128d4d3300f --- /dev/null +++ b/71icjjya/episode_rosbags/episode_3_2024-12-31-23-56-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1d66b4bd75c31fa42906a742e380e3cef582f573d6faddc696bf8185e6c766e +size 2599003175 diff --git a/71icjjya/episode_rosbags/episode_4_2024-12-31-23-58-00.bag b/71icjjya/episode_rosbags/episode_4_2024-12-31-23-58-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..6c9c80c9fddebe58c085d9001bf84bbb4f442504 --- /dev/null +++ b/71icjjya/episode_rosbags/episode_4_2024-12-31-23-58-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a56c15ccdfb66655352af73c0b5b40d368e17f2755cf4a7660f7620d7cc2854 +size 2604779912 diff --git a/7kvn4h3l/.hydra/config.yaml b/7kvn4h3l/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/7kvn4h3l/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7kvn4h3l/.hydra/hydra.yaml b/7kvn4h3l/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0a8ce3bae498d7de472d57e153a657dd55ce080 --- /dev/null +++ b/7kvn4h3l/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7kvn4h3l/.hydra/overrides.yaml b/7kvn4h3l/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7kvn4h3l/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7kvn4h3l/episode_rosbags/aligned_depth_to_color_K.npy b/7kvn4h3l/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7kvn4h3l/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7kvn4h3l/episode_rosbags/cam_tf_world.npy b/7kvn4h3l/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/7kvn4h3l/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/7kvn4h3l/episode_rosbags/color_K.npy b/7kvn4h3l/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7kvn4h3l/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7kvn4h3l/episode_rosbags/depth_K.npy b/7kvn4h3l/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/7kvn4h3l/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/7kvn4h3l/eval_robot.log b/7kvn4h3l/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..adfced0788cbc024e932a2c92297520235d3cb08 --- /dev/null +++ b/7kvn4h3l/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 13:45:37,897][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:45:37,900][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/7kvn4h3l/eval_video/0_eval.mp4 b/7kvn4h3l/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8530e3c13a485a5e556a4185378167b83bde19c2 --- /dev/null +++ b/7kvn4h3l/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f297c584cc71310e0a04b8813546ec8a4736cbd9e998d4167419826e082056c9 +size 1455671 diff --git a/7kvn4h3l/eval_video/1_eval.mp4 b/7kvn4h3l/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1880cad496af3925f1151d990f4d6da59e507745 --- /dev/null +++ b/7kvn4h3l/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6907327e93b8114ee465a2473c3899dc1f9fb8cfa5a94661d128e38086415afc +size 1437308 diff --git a/7kvn4h3l/eval_video/2_eval.mp4 b/7kvn4h3l/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4997117f5c19e441b65fc8876c35206df6d8e2df --- /dev/null +++ b/7kvn4h3l/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4051b4bdb3a6bb8351bac3a5fa655dac50252e9ad943f08b4654ab861e3b7695 +size 1451312 diff --git a/7kvn4h3l/eval_video/3_eval.mp4 b/7kvn4h3l/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ce47807ab31f784ae1d6cffe707caf1f4cb270d --- /dev/null +++ b/7kvn4h3l/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33edf09157ebb5c1fc8b96678db813538d2db1d95580a676f5f281c253320f1d +size 1455027 diff --git a/7kvn4h3l/eval_video/4_eval.mp4 b/7kvn4h3l/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..670e5282e197109a3a2c84850d1c68e0ecb9b71f --- /dev/null +++ b/7kvn4h3l/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca7ef2fca9f922df5d34c2a9177e041da0790b196cbca6389020c9e189878bfd +size 1509739 diff --git a/7kvn4h3l/tb/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 b/7kvn4h3l/tb/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 new file mode 100644 index 0000000000000000000000000000000000000000..d31412b0c747f51cfefa30a71cf7108993c61763 --- /dev/null +++ b/7kvn4h3l/tb/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911810ff85cb98f9ae7452956fb2e5dde8124ab4e2fefe8b82ddf0630edb85c1 +size 6539 diff --git a/7kvn4h3l/wandb/debug-internal.log b/7kvn4h3l/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4183683ed5003db76bd2d7d5c18357afa9ab7587 --- /dev/null +++ b/7kvn4h3l/wandb/debug-internal.log @@ -0,0 +1,831 @@ +2025-01-01 13:45:43,658 INFO StreamThr :1806982 [internal.py:wandb_internal():85] W&B internal server running at pid: 1806982, started at: 2025-01-01 13:45:43.658559 +2025-01-01 13:45:43,660 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:45:43,675 INFO WriterThread:1806982 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb +2025-01-01 13:45:43,675 DEBUG SenderThread:1806982 [sender.py:send():379] send: header +2025-01-01 13:45:43,682 DEBUG SenderThread:1806982 [sender.py:send():379] send: run +2025-01-01 13:45:43,886 INFO SenderThread:1806982 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files +2025-01-01 13:45:43,886 INFO SenderThread:1806982 [sender.py:_start_run_threads():1188] run started: 7kvn4h3l with start time 1735757143.658907 +2025-01-01 13:45:43,903 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:45:43,903 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:45:43,979 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:45:43,984 DEBUG HandlerThread:1806982 [system_info.py:__init__():26] System info init +2025-01-01 13:45:43,984 DEBUG HandlerThread:1806982 [system_info.py:__init__():41] System info init done +2025-01-01 13:45:43,984 INFO HandlerThread:1806982 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:45:43,984 INFO SystemMonitor:1806982 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:45:43,984 INFO HandlerThread:1806982 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:45:43,985 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:45:43,985 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:45:43,986 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:45:43,986 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:45:43,987 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:45:43,997 DEBUG HandlerThread:1806982 [system_info.py:probe():152] Probing system +2025-01-01 13:45:43,998 DEBUG HandlerThread:1806982 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:probe():200] Probing system done +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:45:43.997183', 'startedAt': '2025-01-01T18:45:43.647592', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9913749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.964, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 799.2524108886719}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:45:44,004 INFO HandlerThread:1806982 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:45:44,004 INFO HandlerThread:1806982 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:_save_code():44] Saving code +2025-01-01 13:45:44,006 DEBUG HandlerThread:1806982 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:45:44,006 DEBUG HandlerThread:1806982 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:45:44,068 DEBUG HandlerThread:1806982 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:45:44,068 INFO HandlerThread:1806982 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:45:44,071 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:45:44,305 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/w4guhbkm-wandb-metadata.json +2025-01-01 13:45:44,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:45:44,310 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:45:44,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:44,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:44,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:44,313 INFO HandlerThread:1806982 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/tb" + save: true +} +control { + relay_id: "a11b0b47d46c40d69039f07466f9301d" +} +_info { + stream_id: "7kvn4h3l" +} + +2025-01-01 13:45:44,314 DEBUG HandlerThread:1806982 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:45:44,324 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/nypg5yy3-diff.patch +2025-01-01 13:45:44,379 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/gv62kln3-code/FISH/eval_robot.py +2025-01-01 13:45:44,466 DEBUG SenderThread:1806982 [sender.py:send():379] send: telemetry +2025-01-01 13:45:44,467 DEBUG SenderThread:1806982 [sender.py:send():379] send: tbrecord +2025-01-01 13:45:44,467 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:45:44,467 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 with policy live +2025-01-01 13:45:44,653 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/371z7vqp-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:45:44,887 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH +2025-01-01 13:45:45,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:46,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:46,887 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:47,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:48,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:49,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,460 DEBUG SenderThread:1806982 [sender.py:send():379] send: config +2025-01-01 13:45:50,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:50,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:51,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:52,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:52,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:53,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:54,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:54,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:54,889 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:55,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:56,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:56,890 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:57,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:58,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:58,890 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:59,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:59,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:59,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:59,378 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:00,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:01,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:02,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:02,891 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:03,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:04,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:04,892 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:05,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:05,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:06,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:06,893 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:07,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:08,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:09,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:10,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:10,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:11,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:12,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:13,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:14,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:14,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:14,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:15,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:15,391 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:15,895 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:46:16,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:16,895 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:17,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:18,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:19,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:20,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:21,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:21,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:22,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:23,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:24,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:25,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:26,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:26,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:27,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:28,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:29,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:29,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:29,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:30,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:31,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:31,409 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:32,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:33,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:34,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:35,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:36,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:37,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:37,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:38,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:39,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:40,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:41,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:41,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:46:41,542 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 with policy now +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:46:41,543 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:46:41,783 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/81sr05ti-media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:41,902 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos +2025-01-01 13:46:42,213 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/2f6d55k3-media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:42,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:42,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:42,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:43,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:43,987 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:46:43,987 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:46:44,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:44,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:45,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:46,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:47,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:48,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:48,461 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:49,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:50,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:50,906 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:51,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:52,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:53,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:53,681 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:54,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:54,908 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:55,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:55,909 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:46:56,156 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/ywtr1twl-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:46:56,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:56,909 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:57,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:58,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:59,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:59,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:59,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:59,412 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:00,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:01,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:02,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:02,911 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:03,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:04,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:04,421 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:05,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:06,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:06,913 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:07,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:08,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:09,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:10,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:10,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:11,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:12,336 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:13,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:13,988 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:47:14,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:14,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:14,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:15,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:15,403 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:16,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:17,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:18,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:19,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:20,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:21,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:21,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:22,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:23,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:24,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:25,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:26,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:27,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:27,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:28,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:29,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:29,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:29,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:30,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:31,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:32,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:32,417 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:33,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:34,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:35,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:36,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:36,923 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:37,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:37,619 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:38,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:39,341 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:39,437 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:47:39,437 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:47:39,438 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 with policy now +2025-01-01 13:47:39,438 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:47:39,438 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:39,448 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:39,705 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/grii2ave-media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:39,924 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:47:39,924 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:39,925 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:47:40,209 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/kh2vq79g-media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:40,345 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:40,925 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:41,345 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:42,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:43,347 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:43,462 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:43,988 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:47:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:44,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:45,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:46,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:47,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:48,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:48,927 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:49,017 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:49,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:50,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:51,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:51,928 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:47:52,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:52,753 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:47:52,753 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:52,753 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:52,929 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:47:52,929 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:53,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:54,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:54,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:55,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:56,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:57,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,931 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:59,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:59,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:59,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:59,441 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:00,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:01,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:02,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:02,932 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:03,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:04,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:05,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:05,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:06,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:07,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:08,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:09,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:10,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:11,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:11,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:12,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:13,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:13,989 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:48:14,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:14,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:14,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:15,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:16,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:16,395 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:17,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:18,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:19,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:20,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:21,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:22,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:22,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:23,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:24,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:24,939 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:25,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:26,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:26,940 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:27,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:27,729 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:28,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:29,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:29,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:29,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:30,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:31,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:32,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:33,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:33,471 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:34,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:35,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,489 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,510 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:48:36,510 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 with policy now +2025-01-01 13:48:36,553 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:48:36,554 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:48:36,554 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:36,555 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:36,748 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/tnewkjp8-media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:36,944 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:48:37,230 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/25sicu5t-media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:37,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:38,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:38,555 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:38,944 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:39,496 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:40,497 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:41,494 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:42,493 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:43,496 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:43,989 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:48:43,990 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:44,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:44,319 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:44,495 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:44,946 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:45,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:46,502 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:46,947 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:47,505 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:48,506 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:48,947 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:49,251 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:49,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:49,602 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:48:49,602 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:49,602 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:49,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:48:49,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:48:50,203 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/07252bof-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:48:50,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:50,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:51,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:52,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:53,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:54,308 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:54,508 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:55,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:56,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:56,950 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:57,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:58,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:58,951 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:59,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:59,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:59,442 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:59,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:00,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:00,951 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:01,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:02,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:03,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:04,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:05,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:05,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:06,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:07,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:08,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:09,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:10,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:10,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:11,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:12,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:13,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:13,990 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:49:14,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:14,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:14,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:15,436 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:15,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:16,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:17,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:18,513 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:19,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:20,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:21,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:21,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:22,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:23,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:24,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:25,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:26,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:26,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:27,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:28,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:29,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:29,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:29,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:30,518 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:31,468 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:31,517 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:32,516 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:33,521 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:34,236 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:49:34,237 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 with policy now +2025-01-01 13:49:34,240 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:49:34,241 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:49:34,241 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:34,244 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:34,507 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/88uwtviv-media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:34,522 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:34,964 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:49:35,260 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/jbpby449-media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:35,523 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:36,524 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:36,964 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:37,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:37,524 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:38,522 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:39,526 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:40,536 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:40,965 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:41,533 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:42,527 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:42,966 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:43,069 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:43,532 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:43,990 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:49:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:44,536 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:45,534 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:45,672 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:49:45,672 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:45,673 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:45,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:49:45,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:49:46,534 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:46,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:47,535 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:48,535 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:48,674 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:49,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:50,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:50,968 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:51,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:52,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:52,969 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:53,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:53,855 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:54,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:54,970 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:55,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:56,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:56,971 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:57,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:58,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:59,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:59,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:59,461 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:59,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:00,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:01,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:02,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:03,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:04,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:05,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:05,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:06,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:07,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:08,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:09,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:10,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:10,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:11,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:12,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:13,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:13,991 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:14,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:14,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:14,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:15,455 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:15,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:16,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:17,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:18,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:19,543 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:20,543 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:21,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:21,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:22,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:23,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:24,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:25,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:26,505 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:26,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:26,981 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:27,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:28,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:29,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:29,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:29,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:30,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:30,982 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:31,372 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:31,372 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 with policy now +2025-01-01 13:50:31,377 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:31,378 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:31,378 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:31,380 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:31,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:31,619 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/7btiw71w-media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:31,982 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:31,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:31,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:50:32,264 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/oo6bo152-media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:32,423 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:32,555 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:32,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:33,566 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:34,556 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:35,557 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:36,561 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:37,564 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:38,334 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:38,557 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:39,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:40,565 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:40,986 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:41,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:42,564 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:43,579 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:43,586 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:43,742 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:43,742 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:43,742 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:43,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:43,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:43,991 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:43,999 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:50:44,000 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:50:44,209 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:44,233 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/yajym38l-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:44,458 INFO wandb-upload_0:1806982 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp35sd55vb +2025-01-01 13:50:44,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:44,609 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:50:44,653 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:44,856 INFO SenderThread:1806982 [sender.py:send_request_log_artifact():1525] logged artifact run-7kvn4h3l-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3OTY2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTI0Mw==', 'latestArtifact': None}} +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:44,857 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,858 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,858 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:44,931 DEBUG SenderThread:1806982 [sender.py:send():379] send: telemetry +2025-01-01 13:50:44,931 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,932 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,932 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,932 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,932 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,933 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,933 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,933 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,933 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,934 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,934 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,934 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,934 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,935 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,935 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,935 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,935 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,936 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,936 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,936 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,936 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,937 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,937 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval +2025-01-01 13:50:45,043 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/83woq5mq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:45,198 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/grxoz7p5-media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:45,311 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/2g9sy93w-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:45,312 INFO wandb-upload_1:1806982 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz030igf4 +2025-01-01 13:50:45,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:45,680 INFO SenderThread:1806982 [sender.py:send_request_log_artifact():1525] logged artifact run-7kvn4h3l-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3OTY4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTI0NA==', 'latestArtifact': None}} +2025-01-01 13:50:45,680 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:45,681 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json with policy now +2025-01-01 13:50:45,681 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:45,681 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:45,681 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:45,938 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/ayxov9bj-media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:45,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:46,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:46,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:47,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:48,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:48,682 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:49,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:50,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:51,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:52,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:53,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:54,328 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:54,489 DEBUG SenderThread:1806982 [sender.py:send():379] send: exit +2025-01-01 13:50:54,489 INFO SenderThread:1806982 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_exit():588] handling runtime: 310 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_exit():594] send defer +2025-01-01 13:50:54,490 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,490 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:50:54,490 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:50:54,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:50:54,491 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,491 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:50:54,491 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:50:54,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:50:54,492 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:50:54,492 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:50:54,543 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:50:54,543 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:50:54,544 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:50:54,544 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,544 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:50:54,544 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:50:54,544 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:54,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,544 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:50:54,545 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,545 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:50:54,545 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:50:54,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,545 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:50:54,991 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:56,992 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:00,748 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:00,749 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:00,749 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:51:00,750 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,750 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:51:00,750 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:51:00,750 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,750 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:51:00,751 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send():379] send: summary +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:00,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:51:00,752 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,752 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:51:00,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:51:00,756 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:00,856 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:51:00,856 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,857 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:51:00,857 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,857 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:51:00,994 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:51:00,994 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:51:01,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:04,231 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:51:04,231 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:04,232 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:04,232 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:51:04,232 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:04,232 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:51:04,232 INFO SenderThread:1806982 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:51:04,235 INFO SenderThread:1806982 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:51:04,237 INFO SenderThread:1806982 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:51:04,251 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:51:04,251 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:04,252 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:51:04,251 DEBUG SenderThread:1806982 [sender.py:send():379] send: artifact +2025-01-01 13:51:04,427 INFO SenderThread:1806982 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:51:04,428 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:04,428 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:51:04,428 INFO SenderThread:1806982 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:51:04,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml config.yaml +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json wandb-summary.json +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt requirements.txt +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log output.log +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch diff.patch +2025-01-01 13:51:05,001 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:51:05,005 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,005 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,005 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:51:05,005 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:51:05,204 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:51:05,205 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:51:05,207 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt +2025-01-01 13:51:05,268 INFO wandb-upload_4:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:05,299 INFO wandb-upload_3:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/95r098bz-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:51:05,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,492 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,499 INFO Thread-11 (_thread_body):1806982 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:51:05,499 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,499 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:51:05,500 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,500 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:51:05,500 INFO SenderThread:1806982 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:51:05,501 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:51:05,501 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,501 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:51:05,501 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,502 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:51:05,502 INFO SenderThread:1806982 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:51:05,698 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,698 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:51:05,698 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:51:05,698 DEBUG SenderThread:1806982 [sender.py:send():379] send: final +2025-01-01 13:51:05,699 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,699 DEBUG SenderThread:1806982 [sender.py:send():379] send: footer +2025-01-01 13:51:05,699 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:51:05,699 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,700 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:51:05,700 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,701 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:51:05,701 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:51:05,703 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:51:05,704 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:05,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:51:05,753 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:51:05,754 INFO HandlerThread:1806982 [handler.py:finish():882] shutting down handler +2025-01-01 13:51:06,702 INFO WriterThread:1806982 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [sender.py:finish():1615] shutting down sender +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [file_pusher.py:join():175] waiting for file pusher diff --git a/7kvn4h3l/wandb/debug.log b/7kvn4h3l/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d7ec7bb00c50c86c2e6bf1d73203d9af16844d9d --- /dev/null +++ b/7kvn4h3l/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Configure stats pid to 1806798 +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/settings +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug.log +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug-internal.log +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l'} +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():619] starting backend +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():623] setting up manager +2025-01-01 13:45:43,657 INFO MainThread:1806798 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:45:43,658 INFO MainThread:1806798 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:45:43,674 INFO MainThread:1806798 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:45:43,680 INFO MainThread:1806798 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:45:43,902 INFO MainThread:1806798 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:45:43,974 INFO MainThread:1806798 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:45:43,975 INFO MainThread:1806798 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:45:44,312 INFO MainThread:1806798 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:45:44,313 INFO MainThread:1806798 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/tb, True +2025-01-01 13:45:49,460 INFO MainThread:1806798 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 13:51:06,754 WARNING MsgRouterThr:1806798 [router.py:message_loop():77] message_loop has been closed diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59fd7531de08f5914251c152ee67b09975b5007c --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735757143 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f9cbfaf74a18317e7403f01a665bc5e3d98239c --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json new file mode 100644 index 0000000000000000000000000000000000000000..77a837728c3c3b76eb5bae3942624d62a5082de6 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[24, 55, 0.43636363636363634, 0.28613674644932785, 0.594369336943826, 0.31363795701416297, 0.5759494677034114, 0.33253910395872033, 0.5448034033089676, 0.3477891944653233, 0.5297315381407879]]} \ No newline at end of file diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8530e3c13a485a5e556a4185378167b83bde19c2 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f297c584cc71310e0a04b8813546ec8a4736cbd9e998d4167419826e082056c9 +size 1455671 diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1880cad496af3925f1151d990f4d6da59e507745 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6907327e93b8114ee465a2473c3899dc1f9fb8cfa5a94661d128e38086415afc +size 1437308 diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4997117f5c19e441b65fc8876c35206df6d8e2df --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4051b4bdb3a6bb8351bac3a5fa655dac50252e9ad943f08b4654ab861e3b7695 +size 1451312 diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ce47807ab31f784ae1d6cffe707caf1f4cb270d --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33edf09157ebb5c1fc8b96678db813538d2db1d95580a676f5f281c253320f1d +size 1455027 diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..670e5282e197109a3a2c84850d1c68e0ecb9b71f --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca7ef2fca9f922df5d34c2a9177e041da0790b196cbca6389020c9e189878bfd +size 1509739 diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8b9b94dadec854b4a2848290dd0c26b8db31a4b0 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735757149.461698]: resetting environment +[INFO] [1735757149.469113]: cleared current plan +[INFO] [1735757149.469950]: moving to home +[INFO] [1735757150.472266]: reached home +[INFO] [1735757150.473895]: reset action history +[INFO] [1735757153.184085]: environment reset +Starting episode 0 +[INFO] [1735757153.189003]: resetting environment +[INFO] [1735757153.194021]: cleared current plan +[INFO] [1735757153.195002]: moving to home +[INFO] [1735757154.197263]: reached home +[INFO] [1735757154.198859]: reset action history +[INFO] [1735757156.907535]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735757161.924191]: resetting environment +[INFO] [1735757161.928283]: cleared current plan +[INFO] [1735757161.929765]: moving to home +[INFO] [1735757162.932275]: reached home +[INFO] [1735757162.933681]: reset action history +[INFO] [1735757165.644043]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/episode_rosbags/episode_0_2025-01-01-13-46-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1806798.log +started roslaunch server http://158.130.50.37:40283/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1807089] +process[print_text-2]: started with pid [1807113] +process[rosbag_record-3]: started with pid [1807114] +started bagging! +For topic gripper_width: timestamp difference is -166932708 for nearest: 1735757175586133117 - target: 1735757175753065825 at idx 153 +[INFO] [1735757200.569238]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735757201.317093]: Stored episode 1. +[INFO] [1735757201.317374]: Saving video... +[INFO] [1735757201.498871]: Video saved! +Starting episode 1 +[INFO] [1735757201.511090]: resetting environment +[INFO] [1735757201.522984]: cleared current plan +[INFO] [1735757201.523219]: moving to home +Move to target timeout: 5.099908113479614. Trans error: 0.005063807465846578. Rot error: 0.8668487993315036 +[INFO] [1735757209.926616]: reached home +[INFO] [1735757209.927098]: reset action history +[INFO] [1735757212.637410]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757215.414939]: resetting environment +[INFO] [1735757215.418086]: cleared current plan +[INFO] [1735757215.418242]: moving to home +Move to target timeout: 5.000951766967773. Trans error: 0.005034180722301829. Rot error: 0.8591886359684878 +[INFO] [1735757221.421000]: reached home +[INFO] [1735757221.421422]: reset action history +[INFO] [1735757224.128411]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/episode_rosbags/episode_1_2025-01-01-13-47-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1806798.log +started roslaunch server http://158.130.50.37:40337/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1807249] +process[print_text-5]: started with pid [1807250] +process[rosbag_record-6]: started with pid [1807251] +started bagging! +For topic EE_pose: timestamp difference is 104320303 for nearest: 1735757255060048595 - target: 1735757254955728292 at idx 0 +For topic EE_pose: timestamp difference is 236398266 for nearest: 1735757255192126558 - target: 1735757254955728292 at idx 0 +For topic EE_pose: timestamp difference is 273328167 for nearest: 1735757255229056459 - target: 1735757254955728292 at idx 0 +[INFO] [1735757258.422579]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735757259.185215]: Stored episode 2. +[INFO] [1735757259.185758]: Saving video... +[INFO] [1735757259.390003]: Video saved! +Starting episode 2 +[INFO] [1735757259.436344]: resetting environment +[INFO] [1735757259.460758]: cleared current plan +[INFO] [1735757259.461853]: moving to home +Move to target timeout: 5.099172830581665. Trans error: 0.0051490259556644154. Rot error: 0.9123593969422256 +[INFO] [1735757267.973473]: reached home +[INFO] [1735757267.974248]: reset action history +[INFO] [1735757270.710212]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757271.773841]: resetting environment +[INFO] [1735757271.778160]: cleared current plan +[INFO] [1735757271.778347]: moving to home +Move to target timeout: 5.000588655471802. Trans error: 0.005141621969262818. Rot error: 0.9093164158531732 +[INFO] [1735757277.780588]: reached home +[INFO] [1735757277.781107]: reset action history +[INFO] [1735757280.491574]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/episode_rosbags/episode_2_2025-01-01-13-48-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1806798.log +started roslaunch server http://158.130.50.37:44067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1807390] +process[print_text-8]: started with pid [1807391] +process[rosbag_record-9]: started with pid [1807415] +started bagging! +For topic gripper_width: timestamp difference is 103091231 for nearest: 1735757303852809182 - target: 1735757303749717951 at idx 249 +For topic gripper_width: timestamp difference is 104674256 for nearest: 1735757305686174786 - target: 1735757305581500530 at idx 249 +[INFO] [1735757315.293628]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735757316.223293]: Stored episode 3. +[INFO] [1735757316.225261]: Saving video... +[INFO] [1735757316.478027]: Video saved! +Starting episode 3 +[INFO] [1735757316.515695]: resetting environment +[INFO] [1735757316.526570]: cleared current plan +[INFO] [1735757316.526901]: moving to home +Move to target timeout: 5.003290891647339. Trans error: 0.005161392481263502. Rot error: 0.9000431256053357 +[INFO] [1735757325.137815]: reached home +[INFO] [1735757325.140146]: reset action history +[INFO] [1735757327.919972]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757329.279255]: resetting environment +[INFO] [1735757329.294987]: cleared current plan +[INFO] [1735757329.297434]: moving to home +Move to target timeout: 5.003621816635132. Trans error: 0.005138529639204414. Rot error: 0.8967239621873426 +[INFO] [1735757335.309417]: reached home +[INFO] [1735757335.313388]: reset action history +[INFO] [1735757338.044612]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/episode_rosbags/episode_3_2025-01-01-13-48-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1806798.log +started roslaunch server http://158.130.50.37:33195/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1807534] +process[print_text-11]: started with pid [1807535] +process[rosbag_record-12]: started with pid [1807559] +started bagging! +[INFO] [1735757372.809340]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735757373.630876]: Stored episode 4. +[INFO] [1735757373.635050]: Saving video... +[INFO] [1735757374.199557]: Video saved! +[WARN] [1735757374.145129]: Plan exhausted +[WARN] [1735757374.183539]: Plan exhausted +[WARN] [1735757374.223255]: Plan exhausted +[WARN] [1735757374.263350]: Plan exhausted +Starting episode 4 +[INFO] [1735757374.245779]: resetting environment +[INFO] [1735757374.272099]: cleared current plan +[INFO] [1735757374.277678]: moving to home +[INFO] [1735757379.284781]: reached home +[INFO] [1735757379.290756]: reset action history +[INFO] [1735757382.025785]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757385.026363]: resetting environment +[INFO] [1735757385.055786]: cleared current plan +[INFO] [1735757385.059184]: moving to home +Move to target timeout: 5.0039145946502686. Trans error: 0.005103641011958531. Rot error: 0.8922363317095188 +[INFO] [1735757391.068767]: reached home +[INFO] [1735757391.071216]: reset action history +[INFO] [1735757393.854376]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/episode_rosbags/episode_4_2025-01-01-13-49-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1806798.log +started roslaunch server http://158.130.50.37:44697/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1807710] +process[print_text-14]: started with pid [1807711] +process[rosbag_record-15]: started with pid [1807712] +started bagging! +For topic gripper_width: timestamp difference is 100429911 for nearest: 1735757424452762026 - target: 1735757424352332115 at idx 249 +For topic gripper_width: timestamp difference is 104153741 for nearest: 1735757428886169780 - target: 1735757428782016039 at idx 249 +For topic gripper_width: timestamp difference is 104372753 for nearest: 1735757429152834667 - target: 1735757429048461914 at idx 249 +[INFO] [1735757430.186173]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735757430.930236]: Stored episode 5. +[INFO] [1735757430.934887]: Saving video... +[INFO] [1735757431.315948]: Video saved! +[INFO] [1735757431.378774]: resetting environment +[INFO] [1735757431.420647]: cleared current plan +[INFO] [1735757431.422940]: moving to home +Move to target timeout: 5.005336284637451. Trans error: 0.005177686638697773. Rot error: 0.9126745819463395 +[INFO] [1735757439.849468]: reached home +[INFO] [1735757439.859248]: reset action history +[INFO] [1735757442.646942]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cb66aff8b6d061e2d03330436397c1ee8a54214e --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T18:45:43.997183", + "startedAt": "2025-01-01T18:45:43.647592", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.9913749999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.964, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 799.2524108886719 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..103b98cbe58ca1aa9dafc90af3a092e34541b2cb --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f297c584cc71310e0a04b8813546ec8a4736cbd9e998d4167419826e082056c9", "size": 1455671, "path": "media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4"}, "global_step": 665, "_timestamp": 1735757444.5295546, "_runtime": 300.8706476688385, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6907327e93b8114ee465a2473c3899dc1f9fb8cfa5a94661d128e38086415afc", "size": 1437308, "path": "media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4"}, "eval/num_success": 5.0, "episode": 55.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4051b4bdb3a6bb8351bac3a5fa655dac50252e9ad943f08b4654ab861e3b7695", "size": 1451312, "path": "media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "33edf09157ebb5c1fc8b96678db813538d2db1d95580a676f5f281c253320f1d", "size": 1455027, "path": "media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ca7ef2fca9f922df5d34c2a9177e041da0790b196cbca6389020c9e189878bfd", "size": 1509739, "path": "media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://9ojvd6azqhkr30bnb2tdxq0c1rx6fyclt03oen9aydecr6iy374bbwgfu77bzwn373qsmie0r4ew0zcx4o0onm7qwkydkd0j5rk7zijki7zv24hngqqvq4whdd2a0esg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qag82mlec0g7iiutn9wlv3hrcw3pmh5zru7riicbz9u2193gmjd3cywboat2th1tg068aqc0i75wgxg65ggy726re74y60ur46xebpy24j4ctpc81b0sxh0k3xh4fuiy:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 24, "total_num_episodes": 55, "total_success_rate": 0.43636363636363634, "total_success_rate_lb_0.01": 0.28613674644932785, "total_success_rate_ub_0.01": 0.594369336943826, "total_success_rate_lb_0.025": 0.31363795701416297, "total_success_rate_ub_0.025": 0.5759494677034114, "total_success_rate_lb_0.05": 0.33253910395872033, "total_success_rate_ub_0.05": 0.5448034033089676, "total_success_rate_lb_0.1": 0.3477891944653233, "total_success_rate_ub_0.1": 0.5297315381407879, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "49d35f50ec11ca7504a389ceb05c5b9e10647c29125bac06c83bc27e96521f69", "size": 523, "artifact_path": "wandb-client-artifact://m1q2hywo1jcgz0uscus8v5byj0iqccgdx6uf8z8b8yvi6pir9gxlkcg617goqbt7pgubmn24vyu76k8yzztd61ub5o7j4wdazkq85kyq4chuf49ldps2j4ompqox0gly/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://s062buih9d1sb16feqhxvync2fgy95cxp4n7m0v7t6rf7627dpr2x9zvvsgay1n1e9fvuu8bcx66nmty5mdnwptuowxg7olvcy8iox0zmvonm2gx4ulxl574owiatn24:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 310}, "eval/total_success_rate": 0.4363636374473572, "eval/total_success_rate_lb_0.01": 0.2861367464065552, "eval/total_success_rate_ub_0.01": 0.5943693518638611, "eval/total_success_rate_lb_0.025": 0.31363797187805176, "eval/total_success_rate_ub_0.025": 0.575949490070343, "eval/total_success_rate_lb_0.05": 0.3325391113758087, "eval/total_success_rate_ub_0.05": 0.5448033809661865, "eval/total_success_rate_lb_0.1": 0.3477891981601715, "eval/total_success_rate_ub_0.1": 0.5297315120697021} \ No newline at end of file diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug-internal.log b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4183683ed5003db76bd2d7d5c18357afa9ab7587 --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug-internal.log @@ -0,0 +1,831 @@ +2025-01-01 13:45:43,658 INFO StreamThr :1806982 [internal.py:wandb_internal():85] W&B internal server running at pid: 1806982, started at: 2025-01-01 13:45:43.658559 +2025-01-01 13:45:43,660 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:45:43,675 INFO WriterThread:1806982 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb +2025-01-01 13:45:43,675 DEBUG SenderThread:1806982 [sender.py:send():379] send: header +2025-01-01 13:45:43,682 DEBUG SenderThread:1806982 [sender.py:send():379] send: run +2025-01-01 13:45:43,886 INFO SenderThread:1806982 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files +2025-01-01 13:45:43,886 INFO SenderThread:1806982 [sender.py:_start_run_threads():1188] run started: 7kvn4h3l with start time 1735757143.658907 +2025-01-01 13:45:43,903 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:45:43,903 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:45:43,979 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:45:43,984 DEBUG HandlerThread:1806982 [system_info.py:__init__():26] System info init +2025-01-01 13:45:43,984 DEBUG HandlerThread:1806982 [system_info.py:__init__():41] System info init done +2025-01-01 13:45:43,984 INFO HandlerThread:1806982 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:45:43,984 INFO SystemMonitor:1806982 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:45:43,984 INFO HandlerThread:1806982 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:45:43,985 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:45:43,985 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:45:43,986 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:45:43,986 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:45:43,987 INFO SystemMonitor:1806982 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:45:43,997 DEBUG HandlerThread:1806982 [system_info.py:probe():152] Probing system +2025-01-01 13:45:43,998 DEBUG HandlerThread:1806982 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:probe():200] Probing system done +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:45:43.997183', 'startedAt': '2025-01-01T18:45:43.647592', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9913749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.964, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 799.2524108886719}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:45:44,004 INFO HandlerThread:1806982 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:45:44,004 INFO HandlerThread:1806982 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:45:44,004 DEBUG HandlerThread:1806982 [system_info.py:_save_code():44] Saving code +2025-01-01 13:45:44,006 DEBUG HandlerThread:1806982 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:45:44,006 DEBUG HandlerThread:1806982 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:45:44,068 DEBUG HandlerThread:1806982 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:45:44,068 INFO HandlerThread:1806982 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:45:44,071 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:45:44,072 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:45:44,305 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/w4guhbkm-wandb-metadata.json +2025-01-01 13:45:44,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:45:44,310 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:45:44,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:44,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:44,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:44,313 INFO HandlerThread:1806982 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/tb" + save: true +} +control { + relay_id: "a11b0b47d46c40d69039f07466f9301d" +} +_info { + stream_id: "7kvn4h3l" +} + +2025-01-01 13:45:44,314 DEBUG HandlerThread:1806982 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:45:44,324 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/nypg5yy3-diff.patch +2025-01-01 13:45:44,379 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/gv62kln3-code/FISH/eval_robot.py +2025-01-01 13:45:44,466 DEBUG SenderThread:1806982 [sender.py:send():379] send: telemetry +2025-01-01 13:45:44,467 DEBUG SenderThread:1806982 [sender.py:send():379] send: tbrecord +2025-01-01 13:45:44,467 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:45:44,467 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 with policy live +2025-01-01 13:45:44,653 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/371z7vqp-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:45:44,887 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code +2025-01-01 13:45:44,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH +2025-01-01 13:45:45,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:46,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:46,887 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:47,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:48,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:49,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:49,460 DEBUG SenderThread:1806982 [sender.py:send():379] send: config +2025-01-01 13:45:50,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:50,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:51,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:52,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:52,888 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:53,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:54,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:45:54,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:54,889 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:55,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:56,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:56,890 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:57,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:58,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:58,890 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:45:59,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:45:59,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:45:59,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:45:59,378 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:00,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:01,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:02,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:02,891 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:03,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:04,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:04,892 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:05,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:05,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:06,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:06,893 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:07,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:08,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:09,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:10,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:10,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:11,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:12,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:13,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:14,310 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:14,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:14,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:15,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:15,391 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:15,895 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:46:16,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:16,895 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:17,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:18,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:19,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:20,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:21,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:21,316 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:22,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:23,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:24,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:25,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:26,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:26,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:27,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:28,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:29,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:29,311 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:29,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:30,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:31,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:31,409 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:32,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:33,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:34,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:35,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:36,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:37,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:37,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:38,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:39,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:40,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:41,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:41,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:46:41,542 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 with policy now +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:46:41,542 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:46:41,543 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:46:41,783 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/81sr05ti-media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:41,902 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:46:41,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos +2025-01-01 13:46:42,213 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/2f6d55k3-media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:46:42,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:42,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:42,903 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:43,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:43,987 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:46:43,987 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:46:44,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:44,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:45,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:46,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:47,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:48,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:48,461 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:49,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:50,321 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:50,906 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:51,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:52,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:53,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:53,681 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:46:54,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:54,908 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:55,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:55,909 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:46:56,156 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/ywtr1twl-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:46:56,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:56,909 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:46:57,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:58,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:59,311 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:46:59,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:46:59,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:46:59,412 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:00,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:01,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:02,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:02,911 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:03,323 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:04,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:04,421 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:05,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:06,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:06,913 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:07,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:08,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:09,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:10,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:10,335 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:11,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:12,336 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:13,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:13,988 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:47:14,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:14,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:14,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:15,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:15,403 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:16,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:17,337 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:18,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:19,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:20,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:21,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:21,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:22,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:23,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:24,338 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:25,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:26,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:27,315 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:27,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:28,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:29,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:29,312 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:29,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:30,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:31,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:32,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:32,417 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:33,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:34,339 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:35,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:36,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:36,923 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:37,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:37,619 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:38,340 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:39,341 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:39,437 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:47:39,437 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:47:39,438 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 with policy now +2025-01-01 13:47:39,438 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:47:39,438 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:39,448 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:39,705 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/grii2ave-media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:39,924 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:47:39,924 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:39,925 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:47:40,209 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/kh2vq79g-media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:47:40,345 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:40,925 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:41,345 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:42,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:43,347 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:43,462 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:43,988 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:47:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:44,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:45,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:46,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:47,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:48,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:48,927 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:49,017 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:49,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:50,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:51,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:51,928 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:47:52,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:52,753 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:47:52,753 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:47:52,753 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:47:52,929 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:47:52,929 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:53,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:54,317 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:47:54,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:55,349 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:56,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:57,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,350 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:58,931 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:47:59,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:47:59,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:47:59,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:47:59,441 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:00,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:01,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:02,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:02,932 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:03,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:04,351 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:05,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:05,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:06,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:07,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:08,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:09,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:10,352 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:11,318 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:11,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:12,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:13,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:13,989 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:48:14,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:14,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:14,357 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:15,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:16,353 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:16,395 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:17,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:18,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:19,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:20,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:21,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:22,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:22,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:23,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:24,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:24,939 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:25,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:26,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:26,940 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:27,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:27,729 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:28,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:29,312 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:29,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:29,354 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:30,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:31,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:32,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:33,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:33,471 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:34,355 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:35,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,489 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:36,510 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:48:36,510 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 with policy now +2025-01-01 13:48:36,553 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:48:36,554 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:48:36,554 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:36,555 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:36,748 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/tnewkjp8-media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:36,943 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:36,944 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:48:37,230 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/25sicu5t-media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:48:37,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:38,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:38,555 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:38,944 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:39,496 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:40,497 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:41,494 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:42,493 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:43,496 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:43,989 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:48:43,990 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:44,319 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:44,319 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:44,495 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:44,946 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:45,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:46,502 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:46,947 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:47,505 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:48,506 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:48,947 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:49,251 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:49,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:49,602 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:48:49,602 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:48:49,602 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:48:49,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:48:49,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:48:50,203 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/07252bof-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:48:50,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:50,948 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:51,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:52,507 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:53,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:54,308 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:54,508 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:55,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:56,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:56,950 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:57,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:58,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:48:58,951 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:48:59,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:48:59,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:48:59,442 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:48:59,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:00,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:00,951 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:01,509 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:02,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:03,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:04,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:05,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:05,510 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:06,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:07,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:08,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:09,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:10,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:10,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:11,511 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:12,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:13,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:13,990 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:49:14,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:14,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:14,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:15,436 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:15,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:16,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:17,512 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:18,513 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:19,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:20,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:21,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:21,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:22,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:23,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:24,514 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:25,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:26,320 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:26,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:27,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:28,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:29,313 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:29,313 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:29,515 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:30,518 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:31,468 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:31,517 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:32,516 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:33,521 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:34,236 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:49:34,237 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 with policy now +2025-01-01 13:49:34,240 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:49:34,241 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:49:34,241 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:34,244 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:34,507 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/88uwtviv-media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:34,522 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:34,963 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:34,964 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:49:35,260 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/jbpby449-media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:49:35,523 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:36,524 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:36,964 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:37,322 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:37,524 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:38,522 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:39,526 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:40,536 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:40,965 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:41,533 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:42,527 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:42,966 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:43,069 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:43,532 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:43,990 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:49:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:44,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:44,536 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:45,534 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:45,672 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:49:45,672 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:49:45,673 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:49:45,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:49:45,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:49:46,534 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:46,967 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:47,535 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:48,535 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:48,674 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:49,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:50,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:50,968 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:51,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:52,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:52,969 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:53,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:53,855 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:54,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:54,970 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:55,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:56,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:56,971 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:49:57,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:58,539 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:49:59,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:49:59,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:49:59,461 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:49:59,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:00,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:01,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:02,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:03,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:04,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:05,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:05,540 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:06,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:07,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:08,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:09,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:10,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:10,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:11,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:12,541 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:13,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:13,991 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:14,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:14,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:14,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:15,455 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:15,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:16,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:17,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:18,542 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:19,543 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:20,543 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:21,325 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:21,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:22,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:23,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:24,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:25,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:26,505 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:26,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:26,981 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:27,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:28,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:29,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:29,314 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:29,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:30,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:30,982 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:31,372 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:31,372 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 with policy now +2025-01-01 13:50:31,377 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:31,378 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:31,378 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:31,380 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:31,546 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:31,619 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/7btiw71w-media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:31,982 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:31,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:31,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval +2025-01-01 13:50:32,264 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/oo6bo152-media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:50:32,423 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:32,555 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:32,983 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:33,566 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:34,556 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:35,557 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:36,561 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:37,564 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:38,334 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:38,557 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:39,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:40,565 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:40,986 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:41,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:42,564 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:43,579 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:43,586 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:43,742 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:43,742 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:43,742 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:43,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:43,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:43,991 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:43,999 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:50:44,000 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:50:44,209 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:44,233 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/yajym38l-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:44,314 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:50:44,458 INFO wandb-upload_0:1806982 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp35sd55vb +2025-01-01 13:50:44,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:44,609 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:50:44,653 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:50:44,856 INFO SenderThread:1806982 [sender.py:send_request_log_artifact():1525] logged artifact run-7kvn4h3l-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3OTY2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTI0Mw==', 'latestArtifact': None}} +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:44,857 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:44,857 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,858 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,858 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:50:44,931 DEBUG SenderThread:1806982 [sender.py:send():379] send: telemetry +2025-01-01 13:50:44,931 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,932 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,932 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,932 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,932 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,933 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,933 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,933 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,933 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,934 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,934 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,934 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,934 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,935 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,935 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,935 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,935 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,936 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,936 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,936 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,936 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:44,937 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:44,937 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:50:44,987 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table +2025-01-01 13:50:44,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval +2025-01-01 13:50:45,043 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/83woq5mq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:45,198 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/grxoz7p5-media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:45,311 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/2g9sy93w-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:50:45,312 INFO wandb-upload_1:1806982 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz030igf4 +2025-01-01 13:50:45,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:45,680 INFO SenderThread:1806982 [sender.py:send_request_log_artifact():1525] logged artifact run-7kvn4h3l-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3OTY4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTI0NA==', 'latestArtifact': None}} +2025-01-01 13:50:45,680 DEBUG SenderThread:1806982 [sender.py:send():379] send: files +2025-01-01 13:50:45,681 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json with policy now +2025-01-01 13:50:45,681 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:50:45,681 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:50:45,681 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:45,938 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/ayxov9bj-media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:50:45,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:46,562 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:46,988 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:50:47,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:48,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:48,682 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:49,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:50,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:51,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:52,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:53,563 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:50:54,328 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:50:54,489 DEBUG SenderThread:1806982 [sender.py:send():379] send: exit +2025-01-01 13:50:54,489 INFO SenderThread:1806982 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_exit():588] handling runtime: 310 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_exit():594] send defer +2025-01-01 13:50:54,490 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,490 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:50:54,490 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:50:54,490 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:50:54,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:50:54,491 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,491 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:50:54,491 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:50:54,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:50:54,491 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:50:54,492 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:50:54,492 DEBUG SystemMonitor:1806982 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:50:54,543 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:50:54,543 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:50:54,544 INFO HandlerThread:1806982 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:50:54,544 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,544 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:50:54,544 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:50:54,544 DEBUG SenderThread:1806982 [sender.py:send():379] send: stats +2025-01-01 13:50:54,544 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,544 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:50:54,545 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:50:54,545 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:50:54,545 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:50:54,545 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:50:54,545 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:50:54,991 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:50:56,992 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:00,748 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:00,749 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:00,749 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:51:00,750 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,750 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:51:00,750 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:51:00,750 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,750 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send():379] send: history +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:51:00,751 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:00,751 DEBUG SenderThread:1806982 [sender.py:send():379] send: summary +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:51:00,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:51:00,752 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,752 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:51:00,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,752 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:51:00,756 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:00,856 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:51:00,856 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:00,857 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:51:00,857 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:00,857 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:51:00,994 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:51:00,994 INFO Thread-12 :1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:51:01,491 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:04,231 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:51:04,231 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:04,232 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:04,232 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:51:04,232 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:04,232 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:51:04,232 INFO SenderThread:1806982 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:51:04,235 INFO SenderThread:1806982 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:51:04,237 INFO SenderThread:1806982 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:51:04,251 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:51:04,251 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:04,252 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:51:04,251 DEBUG SenderThread:1806982 [sender.py:send():379] send: artifact +2025-01-01 13:51:04,427 INFO SenderThread:1806982 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:51:04,428 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:04,428 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:51:04,428 INFO SenderThread:1806982 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:51:04,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml config.yaml +2025-01-01 13:51:04,995 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json wandb-summary.json +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt requirements.txt +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log output.log +2025-01-01 13:51:04,996 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/diff.patch diff.patch +2025-01-01 13:51:05,001 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 media/videos/eval/0_eval_0_f297c584cc71310e0a04.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 media/videos/eval/2_eval_2_4051b4bdb3a6bb8351ba.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 media/videos/eval/1_eval_1_6907327e93b8114ee465.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 media/videos/eval/4_eval_4_ca7ef2fca9f922df5d34.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 media/videos/eval/3_eval_3_33edf09157ebb5c1fc8b.mp4 +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:51:05,004 INFO SenderThread:1806982 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json media/table/eval/total_success_rate_ci_6_49d35f50ec11ca7504a3.table.json +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:51:05,005 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,005 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,005 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:51:05,005 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:51:05,005 INFO SenderThread:1806982 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:51:05,204 INFO wandb-upload_2:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/wandb-summary.json +2025-01-01 13:51:05,205 INFO wandb-upload_1:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/config.yaml +2025-01-01 13:51:05,207 INFO wandb-upload_0:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/requirements.txt +2025-01-01 13:51:05,268 INFO wandb-upload_4:1806982 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/files/output.log +2025-01-01 13:51:05,299 INFO wandb-upload_3:1806982 [upload_job.py:push():130] Uploaded file /tmp/tmprk3528t5wandb/95r098bz-events.out.tfevents.1735757144.leonmkim-ROG-Strix-G15CS-G15CS.1806798.0 +2025-01-01 13:51:05,492 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,492 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,499 INFO Thread-11 (_thread_body):1806982 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:51:05,499 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,499 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:51:05,500 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,500 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:51:05,500 INFO SenderThread:1806982 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:51:05,501 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:51:05,501 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,501 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:51:05,501 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,502 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:51:05,502 INFO SenderThread:1806982 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:51:05,698 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,698 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:51:05,698 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:51:05,698 INFO SenderThread:1806982 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:51:05,698 DEBUG SenderThread:1806982 [sender.py:send():379] send: final +2025-01-01 13:51:05,699 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:51:05,699 DEBUG SenderThread:1806982 [sender.py:send():379] send: footer +2025-01-01 13:51:05,699 INFO HandlerThread:1806982 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:51:05,699 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: defer +2025-01-01 13:51:05,700 INFO SenderThread:1806982 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:51:05,700 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,700 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:51:05,701 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:51:05,701 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:51:05,703 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:51:05,704 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:05,752 DEBUG SenderThread:1806982 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:51:05,753 INFO MainThread:1806982 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:51:05,753 DEBUG HandlerThread:1806982 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:51:05,754 INFO HandlerThread:1806982 [handler.py:finish():882] shutting down handler +2025-01-01 13:51:06,702 INFO WriterThread:1806982 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [sender.py:finish():1615] shutting down sender +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:51:06,753 INFO SenderThread:1806982 [file_pusher.py:join():175] waiting for file pusher diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug.log b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d7ec7bb00c50c86c2e6bf1d73203d9af16844d9d --- /dev/null +++ b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Configure stats pid to 1806798 +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/settings +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug.log +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/logs/debug-internal.log +2025-01-01 13:45:43,653 INFO MainThread:1806798 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l'} +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():619] starting backend +2025-01-01 13:45:43,654 INFO MainThread:1806798 [wandb_init.py:init():623] setting up manager +2025-01-01 13:45:43,657 INFO MainThread:1806798 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:45:43,658 INFO MainThread:1806798 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:45:43,674 INFO MainThread:1806798 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:45:43,680 INFO MainThread:1806798 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:45:43,902 INFO MainThread:1806798 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:45:43,974 INFO MainThread:1806798 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:45:43,975 INFO MainThread:1806798 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:45:44,311 INFO MainThread:1806798 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:45:44,312 INFO MainThread:1806798 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:45:44,313 INFO MainThread:1806798 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/7kvn4h3l/tb, True +2025-01-01 13:45:49,460 INFO MainThread:1806798 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 13:51:06,754 WARNING MsgRouterThr:1806798 [router.py:message_loop():77] message_loop has been closed diff --git a/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4bb3efcc12dbd33eae00dd86de34234bc160f111 Binary files /dev/null and b/7kvn4h3l/wandb/run-20250101_134543-7kvn4h3l/run-7kvn4h3l.wandb differ diff --git a/dr7dcnpy/.hydra/config.yaml b/dr7dcnpy/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/dr7dcnpy/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/dr7dcnpy/.hydra/hydra.yaml b/dr7dcnpy/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34d31910d6784ed0935fcf5ee06a5edddddd0a45 --- /dev/null +++ b/dr7dcnpy/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/dr7dcnpy/.hydra/overrides.yaml b/dr7dcnpy/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/dr7dcnpy/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/dr7dcnpy/episode_rosbags/color_K.npy b/dr7dcnpy/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/dr7dcnpy/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/dr7dcnpy/episode_rosbags/episode_0_2024-12-31-23-38-10.bag b/dr7dcnpy/episode_rosbags/episode_0_2024-12-31-23-38-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..ebc870fca79dd42c2833becd430aaae61e52a7fd --- /dev/null +++ b/dr7dcnpy/episode_rosbags/episode_0_2024-12-31-23-38-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4026919fcbac864f0f12e8859bfe0932a03b4d36891eba7a65db07f57aca31db +size 2581329100 diff --git a/dr7dcnpy/episode_rosbags/episode_1_2024-12-31-23-38-59.bag b/dr7dcnpy/episode_rosbags/episode_1_2024-12-31-23-38-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..4715678f68c0883632758f673911ff496b94ae5b --- /dev/null +++ b/dr7dcnpy/episode_rosbags/episode_1_2024-12-31-23-38-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9868c7381dcc833193fcced457dda871de144d8b0031bf20671a2c427baa934 +size 2586991529 diff --git a/dr7dcnpy/episode_rosbags/episode_2_2024-12-31-23-39-50.bag b/dr7dcnpy/episode_rosbags/episode_2_2024-12-31-23-39-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..467e26fe2bb8e424b9581e5a4b970b01e562e638 --- /dev/null +++ b/dr7dcnpy/episode_rosbags/episode_2_2024-12-31-23-39-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:291b992d6a1fbca1ee73f87de363a0b2358421411c51ce543116fba9ec78baf0 +size 2600725426 diff --git a/dr7dcnpy/episode_rosbags/episode_3_2024-12-31-23-40-40.bag b/dr7dcnpy/episode_rosbags/episode_3_2024-12-31-23-40-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..d8175db27bfbc5b0bb60d58ca6aa76ab3bede91d --- /dev/null +++ b/dr7dcnpy/episode_rosbags/episode_3_2024-12-31-23-40-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2aab1f95544d54a912f00a45152d298993a531614e09c2428ed8eab2a4f302 +size 2603976983 diff --git a/dr7dcnpy/episode_rosbags/episode_4_2024-12-31-23-41-30.bag b/dr7dcnpy/episode_rosbags/episode_4_2024-12-31-23-41-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..5cf9c9d14e7c1613268c63dda1cb476e5031cb74 --- /dev/null +++ b/dr7dcnpy/episode_rosbags/episode_4_2024-12-31-23-41-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab5cc89dbb5a1f54e295240022cc9c40a09d5a7a293e99f658c2511935bd501 +size 2600213667 diff --git a/dr7dcnpy/eval_video/0_eval.mp4 b/dr7dcnpy/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77156bfaa3727b4ee4ebadce430a402b1dcf0b47 --- /dev/null +++ b/dr7dcnpy/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8ffd647294593a0f1f078932a0e8231c1b59e6d26ead20b53653b1f03e701d +size 1444532 diff --git a/dr7dcnpy/eval_video/1_eval.mp4 b/dr7dcnpy/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..787bf832320bbdc046edc78a6f14a52ff8f98d36 --- /dev/null +++ b/dr7dcnpy/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc36573cc33918de84227c28494ecebfb00b2e39ad876f07759a7adeece1c9ef +size 1436775 diff --git a/dr7dcnpy/eval_video/2_eval.mp4 b/dr7dcnpy/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37b496fb1bf3eaf899801ad22b91177f845932a9 --- /dev/null +++ b/dr7dcnpy/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde39be34e7ccca2b72c900d6bea5c72e94f4b3e6e922ff3837b2c413135d5f1 +size 1518859 diff --git a/dr7dcnpy/eval_video/3_eval.mp4 b/dr7dcnpy/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95e32eca384cd98ca5db823a8861645508e33d51 --- /dev/null +++ b/dr7dcnpy/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59210e85a219ae050e138be9387b840d6edc9134a8ee967996558441f46da087 +size 1512421 diff --git a/dr7dcnpy/eval_video/4_eval.mp4 b/dr7dcnpy/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6538f94a07ade8620f7b0d1b08df13e9b1ae7346 --- /dev/null +++ b/dr7dcnpy/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61d0a686da651928846548164bb33ca9a48dd9c0e3d46717f186b9841ea88f55 +size 1538555 diff --git a/dr7dcnpy/tb/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 b/dr7dcnpy/tb/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 new file mode 100644 index 0000000000000000000000000000000000000000..f4e3dd8410541401443dec07d6ebf13d76c29dcd --- /dev/null +++ b/dr7dcnpy/tb/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7acdaeb87c346d375b47996615500fbff84bd7421eafca127a8b186e15607b +size 6539 diff --git a/dr7dcnpy/wandb/debug-internal.log b/dr7dcnpy/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b4a9d9b36fb7233e2555a221303076576315e3b1 --- /dev/null +++ b/dr7dcnpy/wandb/debug-internal.log @@ -0,0 +1,788 @@ +2024-12-31 23:37:49,039 INFO StreamThr :1768042 [internal.py:wandb_internal():85] W&B internal server running at pid: 1768042, started at: 2024-12-31 23:37:49.039300 +2024-12-31 23:37:49,040 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:37:49,053 INFO WriterThread:1768042 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb +2024-12-31 23:37:49,054 DEBUG SenderThread:1768042 [sender.py:send():379] send: header +2024-12-31 23:37:49,061 DEBUG SenderThread:1768042 [sender.py:send():379] send: run +2024-12-31 23:37:49,291 INFO SenderThread:1768042 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files +2024-12-31 23:37:49,291 INFO SenderThread:1768042 [sender.py:_start_run_threads():1188] run started: dr7dcnpy with start time 1735706269.039293 +2024-12-31 23:37:49,306 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:37:49,306 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:37:49,383 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:37:49,387 DEBUG HandlerThread:1768042 [system_info.py:__init__():26] System info init +2024-12-31 23:37:49,387 DEBUG HandlerThread:1768042 [system_info.py:__init__():41] System info init done +2024-12-31 23:37:49,387 INFO HandlerThread:1768042 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:37:49,388 INFO SystemMonitor:1768042 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:37:49,388 INFO HandlerThread:1768042 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:37:49,388 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:37:49,389 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:37:49,389 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:37:49,390 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:37:49,390 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:37:49,402 DEBUG HandlerThread:1768042 [system_info.py:probe():152] Probing system +2024-12-31 23:37:49,404 DEBUG HandlerThread:1768042 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:probe():200] Probing system done +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:37:49.402976', 'startedAt': '2025-01-01T04:37:49.031067', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.3723750000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 597.3659858703613}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:37:49,409 INFO HandlerThread:1768042 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:37:49,409 INFO HandlerThread:1768042 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:_save_code():44] Saving code +2024-12-31 23:37:49,411 DEBUG HandlerThread:1768042 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:37:49,411 DEBUG HandlerThread:1768042 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:37:49,472 DEBUG HandlerThread:1768042 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:37:49,473 INFO HandlerThread:1768042 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:37:49,476 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:37:49,708 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:37:49,708 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:37:49,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:37:49,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:37:49,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:49,734 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/1eqdkonx-diff.patch +2024-12-31 23:37:49,735 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/20x5gg49-wandb-metadata.json +2024-12-31 23:37:49,753 INFO HandlerThread:1768042 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/tb" + save: true +} +control { + relay_id: "8c9c1b0b49924e42ad242196d9607f02" +} +_info { + stream_id: "dr7dcnpy" +} + +2024-12-31 23:37:49,753 DEBUG HandlerThread:1768042 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:37:49,772 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/mug2256y-code/FISH/eval_robot.py +2024-12-31 23:37:49,799 DEBUG SenderThread:1768042 [sender.py:send():379] send: telemetry +2024-12-31 23:37:49,804 DEBUG SenderThread:1768042 [sender.py:send():379] send: tbrecord +2024-12-31 23:37:49,804 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:37:49,804 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 with policy live +2024-12-31 23:37:49,998 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/kyv5rqya-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH +2024-12-31 23:37:50,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:51,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:52,292 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:52,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:53,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:54,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:54,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:55,035 DEBUG SenderThread:1768042 [sender.py:send():379] send: config +2024-12-31 23:37:55,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:56,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:56,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:57,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:58,294 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:58,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:59,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:59,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:00,294 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:00,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:01,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:02,295 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:02,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:03,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:04,295 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:04,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:04,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:04,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:04,863 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:05,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:06,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:07,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:08,296 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:08,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:09,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:10,297 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:10,643 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:10,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:11,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:12,298 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:12,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:13,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:14,298 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:14,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:15,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:15,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:16,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:17,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:18,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:19,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:19,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:20,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:20,812 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:21,300 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:38:21,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:22,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:23,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:24,301 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:24,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:25,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:26,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:26,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:27,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:28,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:29,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:30,303 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:30,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:31,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:31,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:32,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:33,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:34,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:34,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:34,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:35,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:36,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:36,863 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:37,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:38,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:39,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:40,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:41,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:42,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:42,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:43,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:44,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:45,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:46,214 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:38:46,214 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 with policy now +2024-12-31 23:38:46,214 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:38:46,215 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:38:46,215 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:38:46,216 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:38:46,307 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:46,307 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos +2024-12-31 23:38:46,509 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/cnrow99o-media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:46,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:46,804 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/0ivfkpe2-media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:47,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:48,223 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:48,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:48,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:49,390 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:38:49,391 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:38:49,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:49,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:49,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:50,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:51,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:52,309 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:52,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:53,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:53,881 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:54,310 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:54,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:55,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:56,311 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:38:56,557 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/1rsw5spg-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:38:56,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:57,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:58,311 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:58,722 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:59,177 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:59,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:00,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:01,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:02,313 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:02,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:03,722 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:04,290 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:04,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:04,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:04,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:05,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:06,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:07,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:08,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:09,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:09,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:10,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:11,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:12,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:13,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:14,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:14,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:15,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:16,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:17,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:18,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:19,391 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:39:19,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:19,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:19,843 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:20,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:21,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:22,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:23,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:24,321 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:24,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:25,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:25,731 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:26,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:27,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:28,731 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:29,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:30,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:31,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:31,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:32,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:33,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:34,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:34,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:34,737 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:35,589 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:39:35,589 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 with policy now +2024-12-31 23:39:35,590 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:39:35,590 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:39:35,590 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:39:35,593 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:39:35,737 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:35,872 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/7qfzw2gh-media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:39:36,607 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/p8hdmlva-media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:37,618 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:37,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:38,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:38,735 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:39,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:40,735 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:41,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:42,335 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:42,736 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:43,349 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:43,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:44,335 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:44,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:45,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:46,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:47,165 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:39:47,165 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:39:47,166 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:39:47,336 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:39:47,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:39:47,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:48,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:48,469 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:48,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:49,392 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:39:49,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:49,714 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:49,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:50,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:50,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:51,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:52,338 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:52,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:53,575 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:53,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:54,339 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:54,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:55,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:56,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:57,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:58,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:58,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:59,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:00,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:01,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:02,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:03,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:03,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:04,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:04,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:04,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:05,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:06,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:07,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:08,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:08,813 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:09,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:10,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:11,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:12,345 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:12,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:13,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:14,669 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:14,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:15,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:16,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:17,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:18,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:19,392 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:40:19,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:19,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:19,837 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:20,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:21,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:22,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:23,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:24,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:25,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:25,745 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:26,745 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:27,749 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:28,119 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:40:28,119 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 with policy now +2024-12-31 23:40:28,120 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:40:28,121 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:40:28,121 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:40:28,132 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,351 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:40:28,376 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/lstfl331-media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,670 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/glunkf01-media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,751 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:29,747 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:30,351 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:30,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:30,752 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:31,749 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:32,751 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:33,754 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:34,352 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:34,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:34,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:34,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:35,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:35,997 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:36,353 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:36,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:37,019 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:40:37,019 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:40:37,019 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:40:37,353 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:40:37,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:40:37,606 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/ii4xxcmi-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:40:37,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:38,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:38,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:39,763 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:40,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:40,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:41,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:41,941 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:42,355 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:42,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:43,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:44,356 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:44,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:45,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:46,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:47,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:47,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:48,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:49,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:40:49,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:49,711 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:49,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:50,358 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:50,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:51,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:52,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:52,865 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:53,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:54,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:55,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:56,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:57,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:58,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:58,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:59,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:00,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:01,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:02,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:03,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:04,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:04,711 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:04,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:04,808 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:05,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:06,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:07,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:08,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:09,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:10,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:10,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:11,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:12,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:13,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:14,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:15,762 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:16,563 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:16,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:17,644 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:41:17,644 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 with policy now +2024-12-31 23:41:17,649 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:41:17,649 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:41:17,650 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:41:17,653 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:41:17,762 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:17,931 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/dnkc55qa-media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,368 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:41:18,664 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/ht6famcx-media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,769 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:19,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:41:19,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:19,726 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:19,773 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:20,368 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:20,779 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:21,590 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:21,773 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:22,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:23,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:24,369 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:24,775 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:25,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:26,370 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:26,718 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:26,777 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:27,370 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:41:27,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:28,084 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:41:28,084 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:41:28,085 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:41:28,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:41:28,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:28,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:29,792 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:30,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:30,783 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:31,789 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:31,997 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:32,372 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:32,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:33,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:34,373 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:34,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:34,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:34,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:35,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:36,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:37,625 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:37,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:38,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:39,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:40,375 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:40,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:41,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:42,718 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:42,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:43,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:44,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:45,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:46,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:47,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:47,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:48,783 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:49,394 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:41:49,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:49,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:49,788 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:50,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:51,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:52,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:52,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:53,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:54,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:55,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:56,380 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:56,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:57,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:58,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:58,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:59,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:00,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:01,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:02,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:03,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:03,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:04,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:42:04,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:42:04,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:05,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:06,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:07,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:08,800 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:08,808 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:09,089 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:09,089 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 with policy now +2024-12-31 23:42:09,093 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:09,093 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:09,093 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:09,106 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:09,354 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/tnpeka16-media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,390 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:09,390 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,391 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:42:09,674 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/46tuzuj7-media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:10,391 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:10,802 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:11,799 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:12,801 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:13,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:14,082 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:14,392 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:14,803 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:15,804 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:16,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:17,803 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:18,393 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:18,802 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:19,394 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:42:19,394 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:19,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:42:19,720 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:42:19,800 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:20,799 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:21,149 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:21,150 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:21,150 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:21,396 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:21,396 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:21,449 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:42:21,450 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:42:21,596 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:21,654 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/tajck34n-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:21,797 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:21,911 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiynw_j2z +2024-12-31 23:42:21,965 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:42:21,975 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval +2024-12-31 23:42:22,404 INFO SenderThread:1768042 [sender.py:send_request_log_artifact():1525] logged artifact run-dr7dcnpy-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzQyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODk2Mw==', 'latestArtifact': None}} +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:22,404 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,405 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,405 DEBUG SenderThread:1768042 [sender.py:send():379] send: telemetry +2024-12-31 23:42:22,405 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,409 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,409 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,410 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,410 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,410 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,410 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,411 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,411 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,411 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,411 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,412 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,412 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,412 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,412 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,413 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,413 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,414 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,414 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:42:22,586 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/qwap38ad-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,619 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/blg4nvaa-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:22,797 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:22,856 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/8byubnre-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,967 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl32othm9 +2024-12-31 23:42:23,333 INFO SenderThread:1768042 [sender.py:send_request_log_artifact():1525] logged artifact run-dr7dcnpy-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzQzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODk2NA==', 'latestArtifact': None}} +2024-12-31 23:42:23,333 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:23,333 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:23,334 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send():379] send: exit +2024-12-31 23:42:23,335 INFO SenderThread:1768042 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:42:23,335 INFO SenderThread:1768042 [sender.py:send_exit():588] handling runtime: 273 +2024-12-31 23:42:23,338 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_exit():594] send defer +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:42:23,339 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:42:23,339 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:42:23,340 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:42:23,340 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:42:23,393 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,393 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:42:23,393 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:42:23,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:42:23,394 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,394 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:42:23,394 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,395 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:42:23,395 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:42:23,395 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,396 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:42:23,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:23,623 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/wlualbed-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:24,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,157 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:42:30,157 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:30,157 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,157 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:42:30,159 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:30,159 DEBUG SenderThread:1768042 [sender.py:send():379] send: summary +2024-12-31 23:42:30,159 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:30,160 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:42:30,160 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,160 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:42:30,160 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:42:30,164 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,264 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:30,294 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:42:30,295 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:30,295 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,295 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:42:30,295 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,295 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:42:30,400 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:42:30,400 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:31,269 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:32,100 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:42:32,101 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:32,101 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:32,101 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:42:32,101 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:32,101 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:42:32,101 INFO SenderThread:1768042 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:42:32,103 INFO SenderThread:1768042 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:42:32,105 INFO SenderThread:1768042 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:42:32,117 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:42:32,117 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:32,118 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:42:32,118 DEBUG SenderThread:1768042 [sender.py:send():379] send: artifact +2024-12-31 23:42:32,270 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:32,401 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:32,510 INFO wandb-upload_0:1768042 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6x_nnfj_ +2024-12-31 23:42:32,510 INFO wandb-upload_1:1768042 [upload_job.py:push():85] Skipped uploading /tmp/tmpoejny5wm/wandb-job.json +2024-12-31 23:42:32,610 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjs_u6hs_ +2024-12-31 23:42:33,201 INFO SenderThread:1768042 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNDA4Mg==', 'versionIndex': 400}}} +2024-12-31 23:42:33,202 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,202 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:42:33,202 INFO SenderThread:1768042 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml config.yaml +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json wandb-summary.json +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt requirements.txt +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log output.log +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch diff.patch +2024-12-31 23:42:33,408 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:42:33,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:33,409 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,411 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:42:33,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,413 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:42:33,413 INFO SenderThread:1768042 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:42:33,597 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt +2024-12-31 23:42:33,626 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:33,647 INFO wandb-upload_4:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:33,650 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:42:33,716 INFO wandb-upload_3:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/8eg6niw0-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:33,917 INFO Thread-11 (_thread_body):1768042 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:42:33,917 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,917 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:42:33,917 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,917 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:42:33,917 INFO SenderThread:1768042 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:42:33,919 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,919 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:42:33,919 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:42:34,109 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:34,109 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:42:34,109 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:42:34,109 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:34,109 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send():379] send: final +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send():379] send: footer +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:34,110 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:42:34,110 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:42:34,111 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:34,111 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:42:34,113 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:42:34,114 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:34,166 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:34,166 INFO MainThread:1768042 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:42:34,167 INFO MainThread:1768042 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:42:34,167 INFO MainThread:1768042 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:42:34,167 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:42:34,167 INFO HandlerThread:1768042 [handler.py:finish():882] shutting down handler +2024-12-31 23:42:35,112 INFO WriterThread:1768042 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [sender.py:finish():1615] shutting down sender +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [file_pusher.py:join():175] waiting for file pusher diff --git a/dr7dcnpy/wandb/debug.log b/dr7dcnpy/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..63844a55bbdd02edc3cd3312363dd72e749d5c73 --- /dev/null +++ b/dr7dcnpy/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Configure stats pid to 1767843 +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/settings +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug.log +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug-internal.log +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy'} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():619] starting backend +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():623] setting up manager +2024-12-31 23:37:49,038 INFO MainThread:1767843 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:37:49,039 INFO MainThread:1767843 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:37:49,053 INFO MainThread:1767843 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:37:49,059 INFO MainThread:1767843 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:37:49,305 INFO MainThread:1767843 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:37:49,375 INFO MainThread:1767843 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:37:49,375 INFO MainThread:1767843 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:37:49,712 INFO MainThread:1767843 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:37:49,712 INFO MainThread:1767843 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/tb, True +2024-12-31 23:37:55,034 INFO MainThread:1767843 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 23:42:35,168 WARNING MsgRouterThr:1767843 [router.py:message_loop():77] message_loop has been closed diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89e86ea76bc115fa4adc5fe8c4fa7192e90cd562 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735706269 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d0fe8478ad7c175b306ffe80f6ea9aaa7ecdf9ce --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..3db3932 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 15.0 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..bf34235b489a6e5ec51578c4d27b7c5aff91730c --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 20, 0.5, 0.24900326137794287, 0.748354494522015, 0.297579273621989, 0.7263675480628274, 0.32785302404031924, 0.6719539165152919, 0.34575230631336995, 0.6572977080626605]]} \ No newline at end of file diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77156bfaa3727b4ee4ebadce430a402b1dcf0b47 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8ffd647294593a0f1f078932a0e8231c1b59e6d26ead20b53653b1f03e701d +size 1444532 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..787bf832320bbdc046edc78a6f14a52ff8f98d36 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc36573cc33918de84227c28494ecebfb00b2e39ad876f07759a7adeece1c9ef +size 1436775 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37b496fb1bf3eaf899801ad22b91177f845932a9 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde39be34e7ccca2b72c900d6bea5c72e94f4b3e6e922ff3837b2c413135d5f1 +size 1518859 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95e32eca384cd98ca5db823a8861645508e33d51 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59210e85a219ae050e138be9387b840d6edc9134a8ee967996558441f46da087 +size 1512421 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6538f94a07ade8620f7b0d1b08df13e9b1ae7346 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61d0a686da651928846548164bb33ca9a48dd9c0e3d46717f186b9841ea88f55 +size 1538555 diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1e5d0aac74bc0b24c9ffd8573dad507cd3733841 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735706275.035327]: resetting environment +[INFO] [1735706275.041193]: cleared current plan +[INFO] [1735706275.042666]: moving to home +[INFO] [1735706276.045266]: reached home +[INFO] [1735706276.046673]: reset action history +[INFO] [1735706278.806966]: environment reset +Starting episode 0 +[INFO] [1735706278.811935]: resetting environment +[INFO] [1735706278.817116]: cleared current plan +[INFO] [1735706278.817904]: moving to home +[INFO] [1735706279.820283]: reached home +[INFO] [1735706279.821645]: reset action history +[INFO] [1735706282.529365]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735706286.922777]: resetting environment +[INFO] [1735706286.926749]: cleared current plan +[INFO] [1735706286.927923]: moving to home +[INFO] [1735706287.930324]: reached home +[INFO] [1735706287.931968]: reset action history +[INFO] [1735706290.642776]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/episode_rosbags/episode_0_2024-12-31-23-38-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1767843.log +started roslaunch server http://158.130.50.37:38587/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1768151] +process[print_text-2]: started with pid [1768175] +process[rosbag_record-3]: started with pid [1768176] +started bagging! +For topic gripper_width: timestamp difference is 108057966 for nearest: 1735706291519949188 - target: 1735706291411891222 at idx 12 +For topic EE_pose: timestamp difference is 175048987 for nearest: 1735706302281703393 - target: 1735706302106654406 at idx 0 +For topic EE_pose: timestamp difference is 253074424 for nearest: 1735706302359728830 - target: 1735706302106654406 at idx 0 +For topic gripper_width: timestamp difference is -215914350 for nearest: 1735706308653281111 - target: 1735706308869195461 at idx 224 +[INFO] [1735706325.269197]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1735706326.009676]: Stored episode 1. +[INFO] [1735706326.010140]: Saving video... +[INFO] [1735706326.205200]: Video saved! +Starting episode 1 +[INFO] [1735706326.216318]: resetting environment +[INFO] [1735706326.222351]: cleared current plan +[INFO] [1735706326.222618]: moving to home +[INFO] [1735706330.128357]: reached home +[INFO] [1735706330.129350]: reset action history +[INFO] [1735706332.839464]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706336.119645]: resetting environment +[INFO] [1735706336.129779]: cleared current plan +[INFO] [1735706336.130870]: moving to home +[INFO] [1735706337.133343]: reached home +[INFO] [1735706337.135297]: reset action history +[INFO] [1735706339.844369]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/episode_rosbags/episode_1_2024-12-31-23-38-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1767843.log +started roslaunch server http://158.130.50.37:35835/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1768321] +process[print_text-5]: started with pid [1768322] +process[rosbag_record-6]: started with pid [1768323] +started bagging! +For topic gripper_width: timestamp difference is -176671305 for nearest: 1735706362453285894 - target: 1735706362629957199 at idx 249 +[INFO] [1735706374.594468]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735706375.339038]: Stored episode 2. +[INFO] [1735706375.340939]: Saving video... +[INFO] [1735706375.556011]: Video saved! +Starting episode 2 +[INFO] [1735706375.592630]: resetting environment +[INFO] [1735706375.615353]: cleared current plan +[INFO] [1735706375.617689]: moving to home +[INFO] [1735706379.527778]: reached home +[INFO] [1735706379.530925]: reset action history +[INFO] [1735706382.305192]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706386.386511]: resetting environment +[INFO] [1735706386.418912]: cleared current plan +[INFO] [1735706386.420261]: moving to home +[INFO] [1735706387.424959]: reached home +[INFO] [1735706387.428457]: reset action history +[INFO] [1735706390.213939]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/episode_rosbags/episode_2_2024-12-31-23-39-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1767843.log +started roslaunch server http://158.130.50.37:40311/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1768464] +process[print_text-8]: started with pid [1768465] +process[rosbag_record-9]: started with pid [1768466] +started bagging! +For topic gripper_width: timestamp difference is 104269810 for nearest: 1735706410619977541 - target: 1735706410515707731 at idx 249 +[INFO] [1735706426.986021]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735706427.844351]: Stored episode 3. +[INFO] [1735706427.850979]: Saving video... +[INFO] [1735706428.071422]: Video saved! +Starting episode 3 +[INFO] [1735706428.122848]: resetting environment +[INFO] [1735706428.153788]: cleared current plan +[INFO] [1735706428.154439]: moving to home +[INFO] [1735706432.057556]: reached home +[INFO] [1735706432.060122]: reset action history +[INFO] [1735706434.952175]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706436.818182]: resetting environment +[INFO] [1735706436.831254]: cleared current plan +[INFO] [1735706436.831452]: moving to home +[INFO] [1735706437.833047]: reached home +[INFO] [1735706437.835880]: reset action history +[INFO] [1735706440.611656]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/episode_rosbags/episode_3_2024-12-31-23-40-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1767843.log +started roslaunch server http://158.130.50.37:33625/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1768605] +process[print_text-11]: started with pid [1768606] +process[rosbag_record-12]: started with pid [1768607] +started bagging! +For topic gripper_width: timestamp difference is -130214138 for nearest: 1735706448253280716 - target: 1735706448383494854 at idx 104 +[INFO] [1735706476.562843]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735706477.376198]: Stored episode 4. +[INFO] [1735706477.380467]: Saving video... +[INFO] [1735706477.590979]: Video saved! +Starting episode 4 +[INFO] [1735706477.652653]: resetting environment +[INFO] [1735706477.687122]: cleared current plan +[INFO] [1735706477.687440]: moving to home +[INFO] [1735706481.589239]: reached home +[INFO] [1735706481.597217]: reset action history +[INFO] [1735706484.403512]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706487.123486]: resetting environment +[INFO] [1735706487.155411]: cleared current plan +[INFO] [1735706487.155735]: moving to home +[INFO] [1735706488.157758]: reached home +[INFO] [1735706488.160853]: reset action history +[INFO] [1735706490.952186]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/episode_rosbags/episode_4_2024-12-31-23-41-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1767843.log +started roslaunch server http://158.130.50.37:38461/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1768744] +process[print_text-14]: started with pid [1768745] +process[rosbag_record-15]: started with pid [1768746] +started bagging! +For topic gripper_width: timestamp difference is 107988650 for nearest: 1735706497620009000 - target: 1735706497512020350 at idx 86 +For topic gripper_width: timestamp difference is -111959178 for nearest: 1735706514053278964 - target: 1735706514165238142 at idx 249 +[INFO] [1735706527.986385]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735706528.807545]: Stored episode 5. +[INFO] [1735706528.816493]: Saving video... +[INFO] [1735706529.032691]: Video saved! +[INFO] [1735706529.100212]: resetting environment +[INFO] [1735706529.123202]: cleared current plan +[INFO] [1735706529.125192]: moving to home +[INFO] [1735706533.035150]: reached home +[INFO] [1735706533.036592]: reset action history +[INFO] [1735706535.838960]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e9a3c603a529359c5c112fb8c10f85626fe781bd --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:37:49.402976", + "startedAt": "2025-01-01T04:37:49.031067", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.3723750000004, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.985, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 597.3659858703613 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..92fe8e9005a578bb20d00803e96b8c516fa93f9c --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "8c8ffd647294593a0f1f078932a0e8231c1b59e6d26ead20b53653b1f03e701d", "size": 1444532, "path": "media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4"}, "global_step": 665, "_timestamp": 1735706541.8569837, "_runtime": 272.8176906108856, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "dc36573cc33918de84227c28494ecebfb00b2e39ad876f07759a7adeece1c9ef", "size": 1436775, "path": "media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "fde39be34e7ccca2b72c900d6bea5c72e94f4b3e6e922ff3837b2c413135d5f1", "size": 1518859, "path": "media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "59210e85a219ae050e138be9387b840d6edc9134a8ee967996558441f46da087", "size": 1512421, "path": "media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "61d0a686da651928846548164bb33ca9a48dd9c0e3d46717f186b9841ea88f55", "size": 1538555, "path": "media/videos/eval/4_eval_4_61d0a686da6519288465.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://ppsfs5aimubsnpn97l96gmdqji4ga8u9lh9rahmfz95ngjvwgl5rnbna4prdpabdv1nrjt9adodm2wnc7cxju6sog9m8052q3xudw4fq6ezdp21c0pf35y1w27scu5hk/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://52nyrc3urh4rz8lvaorvjymey4pmscuhezfjxxh4b491t2a1enup6dxw83ggmta0v0t9l3hmlkl17fw2e9tphwon6k87l9aagplzlt6680ju9j4mv9zmmgncfr7rknoc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 20, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.24900326137794287, "total_success_rate_ub_0.01": 0.748354494522015, "total_success_rate_lb_0.025": 0.297579273621989, "total_success_rate_ub_0.025": 0.7263675480628274, "total_success_rate_lb_0.05": 0.32785302404031924, "total_success_rate_ub_0.05": 0.6719539165152919, "total_success_rate_lb_0.1": 0.34575230631336995, "total_success_rate_ub_0.1": 0.6572977080626605, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "633be57ab8f5d2a03a6ed414b12b88c46f9388363d35325712a36769135ab23a", "size": 506, "artifact_path": "wandb-client-artifact://9w8meev3rb03b4shyoduny77sojfzsc63bi1i8vl5v9da6rfjea80x1k1wal016af2zx523vymjly8iuzsghtjoayqxzgqn5lvcsre64c8ciyh87gl4v3piic4grqnoa/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jy77u14b6bj5gq3hc3oi4azyr7cd5cp2kuh6hms8eqgo3rwzxhdfs7wobmy70i0o1wpe9nxhi3yrlq11lj3jo4ni1sza7icg88edptts4u1qp82z900movjvqx90jwyo:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 273}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.24900326132774353, "eval/total_success_rate_ub_0.01": 0.7483544945716858, "eval/total_success_rate_lb_0.025": 0.297579288482666, "eval/total_success_rate_ub_0.025": 0.7263675332069397, "eval/total_success_rate_lb_0.05": 0.3278530240058899, "eval/total_success_rate_ub_0.05": 0.6719539165496826, "eval/total_success_rate_lb_0.1": 0.3457522988319397, "eval/total_success_rate_ub_0.1": 0.6572977304458618} \ No newline at end of file diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug-internal.log b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b4a9d9b36fb7233e2555a221303076576315e3b1 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug-internal.log @@ -0,0 +1,788 @@ +2024-12-31 23:37:49,039 INFO StreamThr :1768042 [internal.py:wandb_internal():85] W&B internal server running at pid: 1768042, started at: 2024-12-31 23:37:49.039300 +2024-12-31 23:37:49,040 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:37:49,053 INFO WriterThread:1768042 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb +2024-12-31 23:37:49,054 DEBUG SenderThread:1768042 [sender.py:send():379] send: header +2024-12-31 23:37:49,061 DEBUG SenderThread:1768042 [sender.py:send():379] send: run +2024-12-31 23:37:49,291 INFO SenderThread:1768042 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files +2024-12-31 23:37:49,291 INFO SenderThread:1768042 [sender.py:_start_run_threads():1188] run started: dr7dcnpy with start time 1735706269.039293 +2024-12-31 23:37:49,306 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:37:49,306 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:37:49,383 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:37:49,387 DEBUG HandlerThread:1768042 [system_info.py:__init__():26] System info init +2024-12-31 23:37:49,387 DEBUG HandlerThread:1768042 [system_info.py:__init__():41] System info init done +2024-12-31 23:37:49,387 INFO HandlerThread:1768042 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:37:49,388 INFO SystemMonitor:1768042 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:37:49,388 INFO HandlerThread:1768042 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:37:49,388 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:37:49,389 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:37:49,389 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:37:49,390 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:37:49,390 INFO SystemMonitor:1768042 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:37:49,402 DEBUG HandlerThread:1768042 [system_info.py:probe():152] Probing system +2024-12-31 23:37:49,404 DEBUG HandlerThread:1768042 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:probe():200] Probing system done +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:37:49.402976', 'startedAt': '2025-01-01T04:37:49.031067', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.3723750000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 597.3659858703613}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:37:49,409 INFO HandlerThread:1768042 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:37:49,409 INFO HandlerThread:1768042 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:37:49,409 DEBUG HandlerThread:1768042 [system_info.py:_save_code():44] Saving code +2024-12-31 23:37:49,411 DEBUG HandlerThread:1768042 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:37:49,411 DEBUG HandlerThread:1768042 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:37:49,472 DEBUG HandlerThread:1768042 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:37:49,473 INFO HandlerThread:1768042 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:37:49,476 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:37:49,476 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:37:49,708 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:37:49,708 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:37:49,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:37:49,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:37:49,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:49,734 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/1eqdkonx-diff.patch +2024-12-31 23:37:49,735 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/20x5gg49-wandb-metadata.json +2024-12-31 23:37:49,753 INFO HandlerThread:1768042 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/tb" + save: true +} +control { + relay_id: "8c9c1b0b49924e42ad242196d9607f02" +} +_info { + stream_id: "dr7dcnpy" +} + +2024-12-31 23:37:49,753 DEBUG HandlerThread:1768042 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:37:49,772 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/mug2256y-code/FISH/eval_robot.py +2024-12-31 23:37:49,799 DEBUG SenderThread:1768042 [sender.py:send():379] send: telemetry +2024-12-31 23:37:49,804 DEBUG SenderThread:1768042 [sender.py:send():379] send: tbrecord +2024-12-31 23:37:49,804 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:37:49,804 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 with policy live +2024-12-31 23:37:49,998 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/kyv5rqya-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code +2024-12-31 23:37:50,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH +2024-12-31 23:37:50,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:51,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:52,292 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:52,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:53,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:54,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:54,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:55,035 DEBUG SenderThread:1768042 [sender.py:send():379] send: config +2024-12-31 23:37:55,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:56,293 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:56,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:57,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:58,294 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:37:58,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:37:59,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:37:59,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:00,294 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:00,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:01,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:02,295 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:02,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:03,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:04,295 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:04,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:04,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:04,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:04,863 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:05,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:06,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:07,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:08,296 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:08,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:09,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:10,297 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:10,643 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:10,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:11,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:12,298 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:12,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:13,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:14,298 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:14,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:15,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:15,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:16,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:17,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:18,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:19,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:19,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:20,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:20,812 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:21,300 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:38:21,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:22,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:23,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:24,301 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:24,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:25,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:26,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:26,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:27,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:28,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:29,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:30,303 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:30,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:31,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:31,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:32,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:33,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:34,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:34,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:34,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:35,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:36,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:36,863 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:37,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:38,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:39,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:40,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:41,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:42,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:42,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:43,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:44,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:45,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:46,214 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:38:46,214 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 with policy now +2024-12-31 23:38:46,214 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:38:46,215 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:38:46,215 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:38:46,216 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:38:46,307 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:46,307 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:38:46,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos +2024-12-31 23:38:46,509 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/cnrow99o-media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:46,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:46,804 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/0ivfkpe2-media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:38:47,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:48,223 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:48,308 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:48,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:49,390 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:38:49,391 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:38:49,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:38:49,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:38:49,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:50,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:51,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:52,309 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:52,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:53,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:53,881 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:54,310 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:54,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:55,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:56,311 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:38:56,557 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/1rsw5spg-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:38:56,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:57,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:58,311 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:38:58,722 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:38:59,177 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:38:59,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:00,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:01,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:02,313 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:02,721 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:03,722 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:04,290 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:04,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:04,709 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:04,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:05,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:06,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:07,724 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:08,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:09,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:09,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:10,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:11,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:12,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:13,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:14,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:14,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:15,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:16,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:17,726 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:18,727 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:19,391 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:39:19,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:19,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:19,843 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:20,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:21,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:22,728 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:23,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:24,321 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:24,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:25,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:25,731 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:26,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:27,729 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:28,731 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:29,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:30,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:31,712 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:31,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:32,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:33,733 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:34,709 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:34,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:34,737 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:35,589 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:39:35,589 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 with policy now +2024-12-31 23:39:35,590 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:39:35,590 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:39:35,590 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:39:35,593 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:39:35,737 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:35,872 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/7qfzw2gh-media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:39:36,607 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/p8hdmlva-media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:39:36,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:37,618 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:37,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:38,333 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:38,735 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:39,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:40,735 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:41,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:42,335 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:42,736 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:43,349 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:43,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:44,335 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:44,739 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:45,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:46,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:47,165 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:39:47,165 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:39:47,166 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:39:47,336 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:39:47,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:39:47,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:48,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:48,469 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:48,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:49,392 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:39:49,714 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:39:49,714 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:39:49,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:50,337 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:50,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:51,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:52,338 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:52,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:53,575 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:53,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:54,339 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:39:54,740 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:55,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:56,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:57,741 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:58,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:39:58,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:39:59,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:00,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:01,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:02,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:03,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:03,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:04,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:04,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:04,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:05,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:06,742 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:07,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:08,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:08,813 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:09,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:10,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:11,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:12,345 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:12,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:13,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:14,669 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:14,743 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:15,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:16,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:17,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:18,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:19,392 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:40:19,710 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:19,710 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:19,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:19,837 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:20,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:21,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:22,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:23,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:24,744 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:25,715 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:25,745 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:26,745 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:27,749 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:28,119 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:40:28,119 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 with policy now +2024-12-31 23:40:28,120 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:40:28,121 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:40:28,121 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:40:28,132 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:28,350 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,351 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:40:28,376 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/lstfl331-media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,670 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/glunkf01-media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:40:28,751 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:29,747 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:30,351 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:30,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:30,752 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:31,749 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:32,751 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:33,754 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:34,352 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:34,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:34,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:34,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:35,753 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:35,997 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:36,353 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:36,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:37,019 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:40:37,019 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:40:37,019 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:40:37,353 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:40:37,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:40:37,606 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/ii4xxcmi-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:40:37,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:38,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:38,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:39,763 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:40,354 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:40,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:41,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:41,941 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:42,355 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:42,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:43,756 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:44,356 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:44,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:45,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:46,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:47,716 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:47,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:48,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:49,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:40:49,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:40:49,711 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:40:49,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:50,358 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:40:50,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:51,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:52,757 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:52,865 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:53,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:54,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:55,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:56,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:57,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:58,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:40:58,758 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:40:59,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:00,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:01,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:02,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:03,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:04,711 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:04,711 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:04,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:04,808 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:05,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:06,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:07,759 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:08,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:09,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:10,717 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:10,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:11,760 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:12,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:13,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:14,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:15,762 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:16,563 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:16,761 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:17,644 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:41:17,644 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 with policy now +2024-12-31 23:41:17,649 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:41:17,649 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:41:17,650 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:41:17,653 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:41:17,762 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:17,931 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/dnkc55qa-media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:18,367 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,368 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:41:18,664 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/ht6famcx-media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:41:18,769 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:19,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:41:19,725 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:19,726 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:19,773 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:20,368 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:20,779 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:21,590 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:21,773 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:22,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:23,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:24,369 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:24,775 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:25,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:26,370 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:26,718 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:26,777 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:27,370 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:41:27,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:28,084 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:41:28,084 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:41:28,085 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:41:28,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:41:28,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:28,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:29,792 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:30,371 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:30,783 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:31,789 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:31,997 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:32,372 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:32,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:33,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:34,373 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:34,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:34,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:34,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:35,780 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:36,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:37,625 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:37,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:38,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:39,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:40,375 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:40,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:41,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:42,718 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:42,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:43,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:44,781 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:45,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:46,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:47,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:47,782 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:48,783 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:49,394 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:41:49,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:41:49,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:41:49,788 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:50,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:51,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:52,778 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:52,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:53,785 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:54,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:55,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:56,380 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:41:56,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:57,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:58,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:41:58,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:41:59,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:00,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:01,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:02,786 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:03,719 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:03,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:04,713 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:42:04,713 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:42:04,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:05,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:06,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:07,787 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:08,800 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:08,808 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:09,089 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:09,089 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 with policy now +2024-12-31 23:42:09,093 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:09,093 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:09,093 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:09,106 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:09,354 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/tnpeka16-media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,390 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:09,390 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,391 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval +2024-12-31 23:42:09,674 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/46tuzuj7-media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:09,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:10,391 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:10,802 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:11,799 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:12,801 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:13,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:14,082 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:14,392 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:14,803 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:15,804 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:16,796 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:17,803 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:18,393 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:18,802 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:19,394 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:42:19,394 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:19,720 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:42:19,720 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:42:19,800 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:20,799 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:21,149 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:21,150 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:21,150 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:21,396 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:21,396 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:21,449 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:42:21,450 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:42:21,596 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:21,654 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/tajck34n-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:21,797 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:21,911 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiynw_j2z +2024-12-31 23:42:21,965 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:42:21,975 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table +2024-12-31 23:42:22,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval +2024-12-31 23:42:22,404 INFO SenderThread:1768042 [sender.py:send_request_log_artifact():1525] logged artifact run-dr7dcnpy-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzQyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODk2Mw==', 'latestArtifact': None}} +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:22,404 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:22,404 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,405 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,405 DEBUG SenderThread:1768042 [sender.py:send():379] send: telemetry +2024-12-31 23:42:22,405 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,409 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,409 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,410 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,410 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,410 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,410 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,411 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,411 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,411 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,411 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,412 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,412 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,412 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,412 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,413 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,413 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:22,414 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:22,414 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:42:22,586 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/qwap38ad-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,619 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/blg4nvaa-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:22,797 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:22,856 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/8byubnre-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:22,967 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl32othm9 +2024-12-31 23:42:23,333 INFO SenderThread:1768042 [sender.py:send_request_log_artifact():1525] logged artifact run-dr7dcnpy-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzQzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3ODk2NA==', 'latestArtifact': None}} +2024-12-31 23:42:23,333 DEBUG SenderThread:1768042 [sender.py:send():379] send: files +2024-12-31 23:42:23,333 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:23,334 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:23,334 DEBUG SenderThread:1768042 [sender.py:send():379] send: exit +2024-12-31 23:42:23,335 INFO SenderThread:1768042 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:42:23,335 INFO SenderThread:1768042 [sender.py:send_exit():588] handling runtime: 273 +2024-12-31 23:42:23,338 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_exit():594] send defer +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:42:23,339 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:42:23,339 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:42:23,339 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:42:23,339 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,339 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:42:23,340 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:42:23,340 DEBUG SystemMonitor:1768042 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:42:23,340 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:42:23,392 INFO HandlerThread:1768042 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:42:23,393 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,393 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:42:23,393 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:42:23,393 DEBUG SenderThread:1768042 [sender.py:send():379] send: stats +2024-12-31 23:42:23,394 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,394 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:42:23,394 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:23,395 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:42:23,395 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:42:23,395 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:23,396 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:42:23,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:23,623 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/wlualbed-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:24,397 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:42:30,155 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,157 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:42:30,157 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send():379] send: history +2024-12-31 23:42:30,157 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:42:30,157 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,157 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:42:30,159 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:30,159 DEBUG SenderThread:1768042 [sender.py:send():379] send: summary +2024-12-31 23:42:30,159 INFO SenderThread:1768042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:42:30,160 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:42:30,160 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,160 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:42:30,160 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,160 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:42:30,164 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:42:30,264 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:30,294 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:42:30,295 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:30,295 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:30,295 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:42:30,295 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:30,295 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:42:30,400 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:42:30,400 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:31,269 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:32,100 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:42:32,101 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:32,101 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:32,101 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:42:32,101 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:32,101 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:42:32,101 INFO SenderThread:1768042 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:42:32,103 INFO SenderThread:1768042 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:42:32,105 INFO SenderThread:1768042 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:42:32,117 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:42:32,117 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:32,118 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:42:32,118 DEBUG SenderThread:1768042 [sender.py:send():379] send: artifact +2024-12-31 23:42:32,270 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:32,401 INFO Thread-12 :1768042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:32,510 INFO wandb-upload_0:1768042 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6x_nnfj_ +2024-12-31 23:42:32,510 INFO wandb-upload_1:1768042 [upload_job.py:push():85] Skipped uploading /tmp/tmpoejny5wm/wandb-job.json +2024-12-31 23:42:32,610 INFO wandb-upload_2:1768042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjs_u6hs_ +2024-12-31 23:42:33,201 INFO SenderThread:1768042 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNDA4Mg==', 'versionIndex': 400}}} +2024-12-31 23:42:33,202 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,202 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:42:33,202 INFO SenderThread:1768042 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml config.yaml +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json wandb-summary.json +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt requirements.txt +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log output.log +2024-12-31 23:42:33,402 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/diff.patch diff.patch +2024-12-31 23:42:33,408 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 media/videos/eval/3_eval_3_59210e85a219ae050e13.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 media/videos/eval/0_eval_0_8c8ffd647294593a0f1f.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 media/videos/eval/4_eval_4_61d0a686da6519288465.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 media/videos/eval/2_eval_2_fde39be34e7ccca2b72c.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 media/videos/eval/1_eval_1_dc36573cc33918de8422.mp4 +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 23:42:33,409 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:42:33,409 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:33,409 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,411 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:42:33,413 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,413 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:42:33,413 INFO SenderThread:1768042 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:42:33,597 INFO wandb-upload_2:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/requirements.txt +2024-12-31 23:42:33,626 INFO wandb-upload_1:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/wandb-summary.json +2024-12-31 23:42:33,647 INFO wandb-upload_4:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/output.log +2024-12-31 23:42:33,650 INFO wandb-upload_0:1768042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/files/config.yaml +2024-12-31 23:42:33,716 INFO wandb-upload_3:1768042 [upload_job.py:push():130] Uploaded file /tmp/tmpeu4jsf6_wandb/8eg6niw0-events.out.tfevents.1735706269.leonmkim-ROG-Strix-G15CS-G15CS.1767843.0 +2024-12-31 23:42:33,917 INFO Thread-11 (_thread_body):1768042 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:42:33,917 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,917 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:42:33,917 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,917 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:42:33,917 INFO SenderThread:1768042 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:42:33,919 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:33,919 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:42:33,919 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:42:33,919 INFO SenderThread:1768042 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:42:34,109 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:34,109 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:42:34,109 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:42:34,109 INFO SenderThread:1768042 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:42:34,109 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:42:34,109 INFO HandlerThread:1768042 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send():379] send: final +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send():379] send: footer +2024-12-31 23:42:34,110 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: defer +2024-12-31 23:42:34,110 INFO SenderThread:1768042 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:42:34,110 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:42:34,111 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:42:34,111 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:42:34,111 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:42:34,113 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:42:34,114 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:42:34,166 DEBUG SenderThread:1768042 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:42:34,166 INFO MainThread:1768042 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:42:34,167 INFO MainThread:1768042 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:42:34,167 INFO MainThread:1768042 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:42:34,167 DEBUG HandlerThread:1768042 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:42:34,167 INFO HandlerThread:1768042 [handler.py:finish():882] shutting down handler +2024-12-31 23:42:35,112 INFO WriterThread:1768042 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [sender.py:finish():1615] shutting down sender +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:42:35,166 INFO SenderThread:1768042 [file_pusher.py:join():175] waiting for file pusher diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug.log b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..63844a55bbdd02edc3cd3312363dd72e749d5c73 --- /dev/null +++ b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Configure stats pid to 1767843 +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/settings +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug.log +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/logs/debug-internal.log +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy'} +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():619] starting backend +2024-12-31 23:37:49,033 INFO MainThread:1767843 [wandb_init.py:init():623] setting up manager +2024-12-31 23:37:49,038 INFO MainThread:1767843 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:37:49,039 INFO MainThread:1767843 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:37:49,053 INFO MainThread:1767843 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:37:49,059 INFO MainThread:1767843 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:37:49,305 INFO MainThread:1767843 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:37:49,375 INFO MainThread:1767843 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:37:49,375 INFO MainThread:1767843 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:37:49,709 INFO MainThread:1767843 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:37:49,712 INFO MainThread:1767843 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:37:49,712 INFO MainThread:1767843 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/dr7dcnpy/tb, True +2024-12-31 23:37:55,034 INFO MainThread:1767843 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 23:42:35,168 WARNING MsgRouterThr:1767843 [router.py:message_loop():77] message_loop has been closed diff --git a/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb new file mode 100644 index 0000000000000000000000000000000000000000..13bd108ddc0ceed3b725067eb615cce8d37b1b27 Binary files /dev/null and b/dr7dcnpy/wandb/run-20241231_233749-dr7dcnpy/run-dr7dcnpy.wandb differ diff --git a/ey5xebaz/.hydra/config.yaml b/ey5xebaz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/ey5xebaz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ey5xebaz/.hydra/hydra.yaml b/ey5xebaz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f508fc4478c5a3b41a7451d913fbb6f414ea82f --- /dev/null +++ b/ey5xebaz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/ey5xebaz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ey5xebaz/.hydra/overrides.yaml b/ey5xebaz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ey5xebaz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ey5xebaz/eval_policy.log b/ey5xebaz/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..52999bb622227e11a6ca4d2fd75fe7fd42b375a4 --- /dev/null +++ b/ey5xebaz/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-01 13:15:50,628][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:15:50,631][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-01 13:15:51,753][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/h2q92ngg/.hydra/config.yaml b/h2q92ngg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/h2q92ngg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/h2q92ngg/.hydra/hydra.yaml b/h2q92ngg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..630973431f5280ce3eff9f6551ef3db2c23904ed --- /dev/null +++ b/h2q92ngg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/h2q92ngg/.hydra/overrides.yaml b/h2q92ngg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/h2q92ngg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/h2q92ngg/episode_rosbags/aligned_depth_to_color_K.npy b/h2q92ngg/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/h2q92ngg/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/h2q92ngg/episode_rosbags/cam_tf_world.npy b/h2q92ngg/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/h2q92ngg/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/h2q92ngg/episode_rosbags/color_K.npy b/h2q92ngg/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/h2q92ngg/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/h2q92ngg/episode_rosbags/depth_K.npy b/h2q92ngg/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/h2q92ngg/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/h2q92ngg/eval_robot.log b/h2q92ngg/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..4cc2d859ffdeafab29d15c7fbd7739cd49f04664 --- /dev/null +++ b/h2q92ngg/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 13:21:41,212][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:21:41,215][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/h2q92ngg/eval_video/0_eval.mp4 b/h2q92ngg/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e00fb1ed60759c7d3d31ce687633e96795fbac5f --- /dev/null +++ b/h2q92ngg/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df3e06ec6ac885c2c7cf543f605bdbe57356c05be4cfa347a62ea4d4696547d +size 1445779 diff --git a/h2q92ngg/eval_video/1_eval.mp4 b/h2q92ngg/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54ce0897bbdbe3f3dbaaedd09f1c569cee861883 --- /dev/null +++ b/h2q92ngg/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0db46e1e3f7c48ac1b9b545b9e494405624dcd6392c91f18777f69010d566c +size 1455091 diff --git a/h2q92ngg/eval_video/2_eval.mp4 b/h2q92ngg/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..56dce820b1f7bb6da3f213c0eaaae007cf7e53a5 --- /dev/null +++ b/h2q92ngg/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5a41bdc902305f5743472366499ceec05b9c565dca376a22e6a19b6dbf2c3c +size 1415945 diff --git a/h2q92ngg/eval_video/3_eval.mp4 b/h2q92ngg/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e8c741347fee11b071cd621f78423ba2c9069d0 --- /dev/null +++ b/h2q92ngg/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e395aa11844e7b03dc3dfce57a06ff2292167c34f7b063a07692b0913624add +size 1486809 diff --git a/h2q92ngg/eval_video/4_eval.mp4 b/h2q92ngg/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98c0b890e46110b10394e728e8e688855cad52a1 --- /dev/null +++ b/h2q92ngg/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f825f4a9faf4b49846d003ee3899da636134ca7a669f8e12d04eba025ba1fed +size 1514222 diff --git a/h2q92ngg/tb/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 b/h2q92ngg/tb/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 new file mode 100644 index 0000000000000000000000000000000000000000..3e56326978fc500e73c2a96eab172f3e465adfdf --- /dev/null +++ b/h2q92ngg/tb/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ae314230df51f9e5eb0180a2f7657787f30aa253bcb2cf85e19baadaf96329a +size 6539 diff --git a/h2q92ngg/wandb/debug-internal.log b/h2q92ngg/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..52ce5ca1f0e8b3dea6a4b5ddc0a3eb1d89e94cc1 --- /dev/null +++ b/h2q92ngg/wandb/debug-internal.log @@ -0,0 +1,853 @@ +2025-01-01 13:21:46,969 INFO StreamThr :1803652 [internal.py:wandb_internal():85] W&B internal server running at pid: 1803652, started at: 2025-01-01 13:21:46.969049 +2025-01-01 13:21:46,971 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:21:46,982 INFO WriterThread:1803652 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb +2025-01-01 13:21:46,983 DEBUG SenderThread:1803652 [sender.py:send():379] send: header +2025-01-01 13:21:46,993 DEBUG SenderThread:1803652 [sender.py:send():379] send: run +2025-01-01 13:21:47,242 INFO SenderThread:1803652 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files +2025-01-01 13:21:47,242 INFO SenderThread:1803652 [sender.py:_start_run_threads():1188] run started: h2q92ngg with start time 1735755706.969843 +2025-01-01 13:21:47,267 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:21:47,267 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:21:47,311 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:21:47,316 DEBUG HandlerThread:1803652 [system_info.py:__init__():26] System info init +2025-01-01 13:21:47,316 DEBUG HandlerThread:1803652 [system_info.py:__init__():41] System info init done +2025-01-01 13:21:47,316 INFO HandlerThread:1803652 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:21:47,316 INFO SystemMonitor:1803652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:21:47,316 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:21:47,317 INFO HandlerThread:1803652 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:21:47,317 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:21:47,317 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:21:47,318 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:21:47,319 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:21:47,328 DEBUG HandlerThread:1803652 [system_info.py:probe():152] Probing system +2025-01-01 13:21:47,330 DEBUG HandlerThread:1803652 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:probe():200] Probing system done +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:21:47.328956', 'startedAt': '2025-01-01T18:21:46.951055', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.79325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.721, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.934, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.022, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 764.9539833068848}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:21:47,335 INFO HandlerThread:1803652 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:21:47,335 INFO HandlerThread:1803652 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:_save_code():44] Saving code +2025-01-01 13:21:47,341 DEBUG HandlerThread:1803652 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:21:47,341 DEBUG HandlerThread:1803652 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:21:47,403 DEBUG HandlerThread:1803652 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:21:47,404 INFO HandlerThread:1803652 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:21:47,407 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:21:47,407 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:21:47,407 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:21:47,408 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:21:47,659 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:21:47,659 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:21:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:47,666 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/gi0kv4fc-wandb-metadata.json +2025-01-01 13:21:47,688 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/97com0pf-diff.patch +2025-01-01 13:21:47,694 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/trferv11-code/FISH/eval_robot.py +2025-01-01 13:21:47,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:47,705 INFO HandlerThread:1803652 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/tb" + save: true +} +control { + relay_id: "11441bc7840344528928920deeb35cc2" +} +_info { + stream_id: "h2q92ngg" +} + +2025-01-01 13:21:47,705 DEBUG HandlerThread:1803652 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:21:47,730 DEBUG SenderThread:1803652 [sender.py:send():379] send: telemetry +2025-01-01 13:21:47,754 DEBUG SenderThread:1803652 [sender.py:send():379] send: tbrecord +2025-01-01 13:21:47,755 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:21:47,755 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 with policy live +2025-01-01 13:21:47,951 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/yukgbo6b-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH +2025-01-01 13:21:48,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:49,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:50,243 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:21:50,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:51,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:52,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:52,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:53,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:54,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:55,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:56,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:57,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:57,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:58,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:59,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:00,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:01,094 DEBUG SenderThread:1803652 [sender.py:send():379] send: config +2025-01-01 13:22:01,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:02,248 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:02,659 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:02,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:02,796 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:03,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:04,249 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:04,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:05,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:06,249 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:06,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:07,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:08,250 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:08,534 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:08,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:09,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:10,251 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:10,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:11,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:12,251 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:12,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:13,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:13,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:14,252 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:14,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:15,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:16,252 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:16,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:17,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:17,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:18,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:18,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:19,253 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:22:19,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:20,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:21,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:22,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:23,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:24,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:24,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:25,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:26,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:27,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:28,256 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:28,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:29,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:29,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:30,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:31,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:32,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:32,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:33,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:34,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:34,810 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:35,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:36,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:37,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:38,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:39,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:40,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:40,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:41,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:42,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:43,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:44,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:45,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:45,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:46,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:47,319 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:22:47,320 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:22:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:47,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:48,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:49,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:50,025 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:22:50,025 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 with policy now +2025-01-01 13:22:50,026 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:22:50,027 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:22:50,027 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:22:50,027 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:22:50,261 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:22:50,295 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/hwy8tvqf-media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,560 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/2fo4ss65-media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:51,033 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:51,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:52,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:52,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:53,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:54,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:55,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:56,077 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:56,263 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:56,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:57,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:58,264 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:58,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:59,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:00,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:01,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:01,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:02,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:02,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:03,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:04,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:05,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:06,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:06,805 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:07,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:08,267 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:23:08,519 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/hqgrija5-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:23:08,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:09,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:10,268 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:10,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,936 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:12,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:13,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:14,269 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:14,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:15,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:16,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:17,320 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:23:17,321 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:17,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:17,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:18,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:19,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:20,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:21,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:22,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:22,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:23,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:24,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:25,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:26,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:27,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:27,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:28,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:29,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:30,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:31,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:32,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:32,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,721 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:33,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:34,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:35,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:36,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:37,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:38,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:38,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:39,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:40,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:41,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:42,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:43,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:43,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:44,280 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:44,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:45,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:46,678 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:47,321 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:23:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:47,676 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:48,049 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:23:48,049 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 with policy now +2025-01-01 13:23:48,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:23:48,053 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:23:48,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:23:48,058 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,282 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:23:48,321 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/wj8s672l-media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,628 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/buvd4j6t-media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,677 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:49,073 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:49,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:50,282 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:50,686 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:51,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:52,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:53,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:54,283 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:54,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:54,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:55,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:56,284 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:56,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:57,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:58,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:59,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:00,405 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:00,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:01,135 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:24:01,136 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:01,136 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:01,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:24:01,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:24:01,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:02,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:02,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:02,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:03,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:04,287 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:04,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:05,406 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:05,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:06,287 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:06,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:07,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:08,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:09,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:10,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:10,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:11,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:12,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:13,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:14,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:15,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:15,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:16,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:17,321 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:24:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:17,661 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:17,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:18,291 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:18,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:19,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:20,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:20,761 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:21,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:22,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:23,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:24,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:25,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:26,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:26,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:27,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:28,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:29,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:30,295 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:30,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:31,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,296 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:32,401 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,701 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:33,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:34,297 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:34,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:35,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:36,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:37,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:37,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:38,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:38,686 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:38,746 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:24:38,746 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 with policy now +2025-01-01 13:24:38,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:24:38,747 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:24:38,748 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:38,755 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:39,058 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/2vinot8v-media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:24:39,586 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/yzu18ye1-media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:40,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:40,693 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:41,692 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:42,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:42,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:43,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:44,300 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:44,698 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:45,721 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:46,691 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:47,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:24:47,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:47,661 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:47,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:47,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:48,301 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:48,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:49,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:50,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:51,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:52,703 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:53,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:53,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:54,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:55,698 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:56,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:57,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:58,700 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:59,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:59,699 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:00,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:01,700 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:02,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:02,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:02,709 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:03,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:04,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:04,817 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:05,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:06,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:07,704 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:08,708 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:09,712 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:10,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:10,712 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:11,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:12,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:13,707 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:14,709 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:15,677 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:15,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:16,710 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:17,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:25:17,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:17,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:17,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:18,059 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:25:18,059 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:18,059 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:18,311 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:25:18,311 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:25:18,579 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/7das7btf-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:25:18,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:19,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:20,312 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:20,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:21,041 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:21,719 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:22,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:23,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:24,313 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:24,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:25,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:26,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:26,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:27,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:28,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:29,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:30,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:31,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:31,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:32,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:32,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:32,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:33,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:34,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:35,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:36,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:36,793 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:37,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:38,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:39,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:40,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:41,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:42,588 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:42,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:43,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:44,320 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:44,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:45,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:46,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:47,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:25:47,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:47,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:47,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:47,800 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:48,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:49,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:50,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:51,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:52,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:53,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:53,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:54,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:55,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:56,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:57,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:58,055 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:25:58,055 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 with policy now +2025-01-01 13:25:58,058 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:25:58,059 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:25:58,059 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:58,059 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:58,324 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:25:58,348 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/dv8zver5-media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,627 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/vh1tv1dz-media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,726 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:59,074 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:59,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:00,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:00,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:01,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:02,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:02,687 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:02,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:03,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:04,327 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:04,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:04,772 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:05,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:06,327 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:26:06,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:06,727 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:07,060 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:26:07,061 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:07,082 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:07,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:26:07,732 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:08,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:08,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:09,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:09,955 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:10,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:11,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:12,329 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:12,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:13,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:14,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:15,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:15,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:16,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:17,323 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:26:17,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:17,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:17,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:18,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:19,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:20,332 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:20,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:21,546 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:21,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:22,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:23,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:24,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:25,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:26,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:26,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:27,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:28,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:29,731 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:30,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:31,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:32,304 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:32,336 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:32,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:32,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:32,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:33,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:34,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:35,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:36,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:37,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:37,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:38,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:39,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:40,339 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:40,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:41,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:42,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:42,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:43,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:44,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:45,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:46,305 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:26:46,305 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 with policy now +2025-01-01 13:26:46,306 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:26:46,306 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:26:46,306 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:46,308 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:46,342 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:26:46,590 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/jcss2q0w-media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:46,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:46,874 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/zx1veo6m-media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:47,323 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:26:47,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:47,670 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:47,739 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:47,779 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:48,342 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:48,743 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:49,746 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:50,744 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:51,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:52,343 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:52,751 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:53,248 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:53,750 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:54,344 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:54,749 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:55,754 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:56,754 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:57,758 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:58,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:58,759 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:59,756 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:00,756 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:01,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:02,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:02,675 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:02,761 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:03,746 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:03,765 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:04,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:05,762 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:06,762 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:07,772 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:08,765 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:09,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:09,768 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:10,349 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:10,592 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/6ttli542-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:10,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:27:10,715 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:27:10,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:10,928 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:11,165 INFO wandb-upload_0:1803652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp270bkk4f +2025-01-01 13:27:11,318 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:27:11,327 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media +2025-01-01 13:27:11,351 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval +2025-01-01 13:27:11,351 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table +2025-01-01 13:27:11,530 INFO SenderThread:1803652 [sender.py:send_request_log_artifact():1525] logged artifact run-h2q92ngg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc2NDY0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1MzE1Nw==', 'latestArtifact': None}} +2025-01-01 13:27:11,530 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:27:11,530 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-01 13:27:11,532 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:11,532 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,535 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,535 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:11,535 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,536 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send():379] send: telemetry +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,536 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,537 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,537 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,537 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,538 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,538 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,538 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,538 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,539 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,539 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,539 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,539 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,540 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,540 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,541 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,541 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,541 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,541 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:27:11,717 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/csko8po3-media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:11,763 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:11,780 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/wqj5tiix-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:11,912 INFO wandb-upload_0:1803652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6o5mgxom +2025-01-01 13:27:12,035 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/8tjeydme-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:12,294 INFO SenderThread:1803652 [sender.py:send_request_log_artifact():1525] logged artifact run-h2q92ngg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc2NDY1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1MzE1OQ==', 'latestArtifact': None}} +2025-01-01 13:27:12,294 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:27:12,294 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json with policy now +2025-01-01 13:27:12,295 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:12,295 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:12,295 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:12,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:12,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:12,546 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/mqf1uwvf-media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:12,763 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:13,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:14,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:15,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:15,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:16,493 DEBUG SenderThread:1803652 [sender.py:send():379] send: exit +2025-01-01 13:27:16,493 INFO SenderThread:1803652 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:27:16,494 INFO SenderThread:1803652 [sender.py:send_exit():588] handling runtime: 329 +2025-01-01 13:27:16,495 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:16,495 INFO SenderThread:1803652 [sender.py:send_exit():594] send defer +2025-01-01 13:27:16,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,495 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:27:16,495 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:27:16,496 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:27:16,496 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:27:16,496 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:27:16,497 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:27:16,497 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:27:16,497 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:27:16,497 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:27:16,551 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,551 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:27:16,551 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,551 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:27:17,352 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:18,353 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:27:23,145 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:23,146 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:23,147 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:27:23,147 DEBUG SenderThread:1803652 [sender.py:send():379] send: summary +2025-01-01 13:27:23,151 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:23,151 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,151 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:27:23,152 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:27:23,152 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,152 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:27:23,152 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,152 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:27:23,157 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:23,406 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:27:23,406 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,406 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:27:23,406 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,406 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:27:23,407 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:23,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:24,161 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:27:24,161 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:24,162 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,162 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:27:24,162 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,162 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:27:24,162 INFO SenderThread:1803652 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:27:24,164 INFO SenderThread:1803652 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:27:24,166 INFO SenderThread:1803652 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:27:24,183 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:27:24,183 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,183 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:27:24,183 DEBUG SenderThread:1803652 [sender.py:send():379] send: artifact +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:27:24,365 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:27:24,407 INFO SenderThread:1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml config.yaml +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json wandb-summary.json +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt requirements.txt +2025-01-01 13:27:24,411 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log output.log +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch diff.patch +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:27:24,417 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,418 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:27:24,418 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,418 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:27:24,418 INFO SenderThread:1803652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:27:24,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:24,495 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:24,603 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:27:24,619 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:24,648 INFO wandb-upload_3:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt +2025-01-01 13:27:24,662 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/qxbcjdun-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:24,671 INFO wandb-upload_4:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:24,871 INFO Thread-11 (_thread_body):1803652 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:27:24,871 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,872 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:27:24,872 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,872 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:27:24,872 INFO SenderThread:1803652 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:27:24,873 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,873 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:27:24,873 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:27:25,051 INFO SenderThread:1803652 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:27:25,052 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:25,052 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send():379] send: final +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send():379] send: footer +2025-01-01 13:27:25,052 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:25,053 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:27:25,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:25,053 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:27:25,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:25,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:27:25,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:25,054 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:25,054 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:27:25,054 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:27:25,055 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:27:25,057 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:25,105 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:27:25,107 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:27:25,107 INFO HandlerThread:1803652 [handler.py:finish():882] shutting down handler +2025-01-01 13:27:26,055 INFO WriterThread:1803652 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [sender.py:finish():1615] shutting down sender +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [file_pusher.py:join():175] waiting for file pusher diff --git a/h2q92ngg/wandb/debug.log b/h2q92ngg/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac36e1365603f790f2d3e0c7a4d5ea6bb16225a0 --- /dev/null +++ b/h2q92ngg/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Configure stats pid to 1803478 +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/settings +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug.log +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug-internal.log +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg'} +2025-01-01 13:21:46,960 INFO MainThread:1803478 [wandb_init.py:init():619] starting backend +2025-01-01 13:21:46,960 INFO MainThread:1803478 [wandb_init.py:init():623] setting up manager +2025-01-01 13:21:46,968 INFO MainThread:1803478 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:21:46,969 INFO MainThread:1803478 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:21:46,982 INFO MainThread:1803478 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:21:46,990 INFO MainThread:1803478 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:21:47,267 INFO MainThread:1803478 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:21:47,306 INFO MainThread:1803478 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:21:47,306 INFO MainThread:1803478 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:21:47,662 INFO MainThread:1803478 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:21:47,663 INFO MainThread:1803478 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/tb, True +2025-01-01 13:22:01,094 INFO MainThread:1803478 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 13:27:26,107 WARNING MsgRouterThr:1803478 [router.py:message_loop():77] message_loop has been closed diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c3a41fa45e88cd6bc6530832c9e008ebc717905 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735755706 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f9cbfaf74a18317e7403f01a665bc5e3d98239c --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json new file mode 100644 index 0000000000000000000000000000000000000000..04704a02fc8db2b3f693744f09b5fdd13dde6aa2 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[18, 40, 0.45, 0.2738732099985334, 0.6341655254095682, 0.306664362588449, 0.614022277272785, 0.3285517841920427, 0.576350063070286, 0.34494671973706914, 0.560576409089371]]} \ No newline at end of file diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e00fb1ed60759c7d3d31ce687633e96795fbac5f --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df3e06ec6ac885c2c7cf543f605bdbe57356c05be4cfa347a62ea4d4696547d +size 1445779 diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54ce0897bbdbe3f3dbaaedd09f1c569cee861883 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0db46e1e3f7c48ac1b9b545b9e494405624dcd6392c91f18777f69010d566c +size 1455091 diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..56dce820b1f7bb6da3f213c0eaaae007cf7e53a5 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5a41bdc902305f5743472366499ceec05b9c565dca376a22e6a19b6dbf2c3c +size 1415945 diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e8c741347fee11b071cd621f78423ba2c9069d0 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e395aa11844e7b03dc3dfce57a06ff2292167c34f7b063a07692b0913624add +size 1486809 diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98c0b890e46110b10394e728e8e688855cad52a1 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f825f4a9faf4b49846d003ee3899da636134ca7a669f8e12d04eba025ba1fed +size 1514222 diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0c6fc58ba2955d9755c1fad8bbae5d618579f1bb --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log @@ -0,0 +1,296 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735755721.094954]: resetting environment +[INFO] [1735755721.099517]: cleared current plan +[INFO] [1735755721.100364]: moving to home +[INFO] [1735755722.102571]: reached home +[INFO] [1735755722.103744]: reset action history +[INFO] [1735755724.812296]: environment reset +Starting episode 0 +[INFO] [1735755724.816626]: resetting environment +[INFO] [1735755724.820634]: cleared current plan +[INFO] [1735755724.821510]: moving to home +[INFO] [1735755725.824364]: reached home +[INFO] [1735755725.825839]: reset action history +[INFO] [1735755728.533395]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735755730.438341]: resetting environment +[INFO] [1735755730.445387]: cleared current plan +[INFO] [1735755730.446504]: moving to home +[INFO] [1735755731.449261]: reached home +[INFO] [1735755731.450518]: reset action history +[INFO] [1735755734.209708]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/episode_rosbags/episode_0_2025-01-01-13-22-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1803478.log +started roslaunch server http://158.130.50.37:45603/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1803781] +process[print_text-2]: started with pid [1803782] +process[rosbag_record-3]: started with pid [1803783] +started bagging! +For topic gripper_width: timestamp difference is 111462520 for nearest: 1735755745952825950 - target: 1735755745841363430 at idx 168 +[INFO] [1735755769.040786]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735755769.807037]: Stored episode 1. +[INFO] [1735755769.808618]: Saving video... +[INFO] [1735755770.015397]: Video saved! +Starting episode 1 +[INFO] [1735755770.026401]: resetting environment +[INFO] [1735755770.032351]: cleared current plan +[INFO] [1735755770.032912]: moving to home +[INFO] [1735755774.036853]: reached home +[INFO] [1735755774.039676]: reset action history +[INFO] [1735755776.747552]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735755788.217359]: resetting environment +[INFO] [1735755788.222621]: cleared current plan +[INFO] [1735755788.223159]: moving to home +[INFO] [1735755789.225306]: reached home +[INFO] [1735755789.226369]: reset action history +[INFO] [1735755791.935301]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/episode_rosbags/episode_1_2025-01-01-13-23-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1803478.log +started roslaunch server http://158.130.50.37:42739/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1803907] +process[print_text-5]: started with pid [1803908] +process[rosbag_record-6]: started with pid [1803932] +started bagging! +For topic gripper_width: timestamp difference is 106094965 for nearest: 1735755821952826866 - target: 1735755821846731901 at idx 249 +[INFO] [1735755826.595694]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735755827.429370]: Stored episode 2. +[INFO] [1735755827.429877]: Saving video... +[INFO] [1735755828.023358]: Video saved! +Starting episode 2 +[INFO] [1735755828.055601]: resetting environment +[INFO] [1735755828.069840]: cleared current plan +[INFO] [1735755828.072494]: moving to home +[WARN] [1735755827.937443]: Plan exhausted +[WARN] [1735755827.972537]: Plan exhausted +[WARN] [1735755828.013700]: Plan exhausted +[WARN] [1735755828.052479]: Plan exhausted +[INFO] [1735755832.180551]: reached home +[INFO] [1735755832.184859]: reset action history +[INFO] [1735755834.916154]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735755840.416005]: resetting environment +[INFO] [1735755840.423401]: cleared current plan +[INFO] [1735755840.423981]: moving to home +[INFO] [1735755841.425327]: reached home +[INFO] [1735755841.426188]: reset action history +[INFO] [1735755844.137422]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/episode_rosbags/episode_2_2025-01-01-13-24-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1803478.log +started roslaunch server http://158.130.50.37:45743/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1804053] +process[print_text-8]: started with pid [1804054] +process[rosbag_record-9]: started with pid [1804078] +started bagging! +For topic gripper_width: timestamp difference is 100480780 for nearest: 1735755856252823338 - target: 1735755856152342558 at idx 169 +For topic gripper_width: timestamp difference is 103732340 for nearest: 1735755868452848904 - target: 1735755868349116564 at idx 249 +For topic gripper_width: timestamp difference is 103217551 for nearest: 1735755871352820584 - target: 1735755871249603033 at idx 249 +For topic gripper_width: timestamp difference is 105940075 for nearest: 1735755872652821035 - target: 1735755872546880960 at idx 249 +preempting episode +[INFO] [1735755877.232143]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1735755878.033481]: Stored episode 3. +[INFO] [1735755878.038322]: Saving video... +[INFO] [1735755878.716779]: Video saved! +Starting episode 3 +[INFO] [1735755878.751247]: resetting environment +[INFO] [1735755878.778580]: cleared current plan +[INFO] [1735755878.778777]: moving to home +[WARN] [1735755878.452680]: Plan exhausted +[WARN] [1735755878.493525]: Plan exhausted +[WARN] [1735755878.532228]: Plan exhausted +[WARN] [1735755878.572206]: Plan exhausted +[WARN] [1735755878.617648]: Plan exhausted +[WARN] [1735755878.654135]: Plan exhausted +[WARN] [1735755878.697802]: Plan exhausted +[WARN] [1735755878.732192]: Plan exhausted +[WARN] [1735755878.772373]: Plan exhausted +[INFO] [1735755882.781342]: reached home +[INFO] [1735755882.784144]: reset action history +[INFO] [1735755885.677631]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735755917.986248]: resetting environment +[INFO] [1735755917.996466]: cleared current plan +[INFO] [1735755917.996830]: moving to home +[INFO] [1735755918.998867]: reached home +[INFO] [1735755919.002167]: reset action history +[INFO] [1735755921.778664]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/episode_rosbags/episode_3_2025-01-01-13-25-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1803478.log +started roslaunch server http://158.130.50.37:46235/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1804194] +process[print_text-11]: started with pid [1804195] +process[rosbag_record-12]: started with pid [1804219] +started bagging! +For topic gripper_width: timestamp difference is 103659214 for nearest: 1735755942586179079 - target: 1735755942482519865 at idx 249 +For topic gripper_width: timestamp difference is 103849859 for nearest: 1735755942852822037 - target: 1735755942748972178 at idx 249 +[INFO] [1735755956.607807]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735755957.341803]: Stored episode 4. +[INFO] [1735755957.343974]: Saving video... +[INFO] [1735755958.027995]: Video saved! +Starting episode 4 +[INFO] [1735755958.060565]: resetting environment +[INFO] [1735755958.074139]: cleared current plan +[INFO] [1735755958.074346]: moving to home +[WARN] [1735755957.973443]: Plan exhausted +[WARN] [1735755958.017389]: Plan exhausted +[WARN] [1735755958.052957]: Plan exhausted +[INFO] [1735755962.177397]: reached home +[INFO] [1735755962.184011]: reset action history +[INFO] [1735755964.957865]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735755966.168726]: resetting environment +[INFO] [1735755966.214126]: cleared current plan +[INFO] [1735755966.215957]: moving to home +[INFO] [1735755967.218894]: reached home +[INFO] [1735755967.230131]: reset action history +[INFO] [1735755969.954704]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/episode_rosbags/episode_4_2025-01-01-13-26-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1803478.log +started roslaunch server http://158.130.50.37:35593/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1804336] +process[print_text-14]: started with pid [1804337] +process[rosbag_record-15]: started with pid [1804361] +started bagging! +For topic gripper_width: timestamp difference is -106829343 for nearest: 1735755970586102786 - target: 1735755970692932129 at idx 9 +For topic gripper_width: timestamp difference is -134104424 for nearest: 1735755978286181516 - target: 1735755978420285940 at idx 117 +For topic gripper_width: timestamp difference is 109573821 for nearest: 1735755991252820710 - target: 1735755991143246889 at idx 249 +For topic gripper_width: timestamp difference is 115939330 for nearest: 1735755999086211633 - target: 1735755998970272303 at idx 249 +[INFO] [1735756004.791616]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735756005.557294]: Stored episode 5. +[INFO] [1735756005.567363]: Saving video... +[INFO] [1735756006.261046]: Video saved! +[INFO] [1735756006.306970]: resetting environment +[INFO] [1735756006.355620]: cleared current plan +[INFO] [1735756006.355858]: moving to home +[WARN] [1735756006.137414]: Plan exhausted +[WARN] [1735756006.173784]: Plan exhausted +[WARN] [1735756006.212909]: Plan exhausted +[WARN] [1735756006.252199]: Plan exhausted +[WARN] [1735756006.292189]: Plan exhausted +[WARN] [1735756006.336120]: Plan exhausted +[INFO] [1735756010.462209]: reached home +[INFO] [1735756010.467279]: reset action history +[INFO] [1735756013.247361]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fd4adfd50aae06b772cb4284a0c14dd8534b9a06 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T18:21:47.328956", + "startedAt": "2025-01-01T18:21:46.951055", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.79325, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.721, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.934, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.022, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 764.9539833068848 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e23e1d46cd868bbd9ea9d47cdc444e75e6972ee --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "4df3e06ec6ac885c2c7cf543f605bdbe57356c05be4cfa347a62ea4d4696547d", "size": 1445779, "path": "media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4"}, "global_step": 658, "_timestamp": 1735756031.2354076, "_runtime": 324.2655646800995, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "dc0db46e1e3f7c48ac1b9b545b9e494405624dcd6392c91f18777f69010d566c", "size": 1455091, "path": "media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4"}, "eval/num_success": 4.0, "episode": 40.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "1c5a41bdc902305f5743472366499ceec05b9c565dca376a22e6a19b6dbf2c3c", "size": 1415945, "path": "media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2e395aa11844e7b03dc3dfce57a06ff2292167c34f7b063a07692b0913624add", "size": 1486809, "path": "media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "3f825f4a9faf4b49846d003ee3899da636134ca7a669f8e12d04eba025ba1fed", "size": 1514222, "path": "media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://gb746kvogpcu553j26hdplxavfyfiz405ztd8uz0wu1kmwpmgjq34oo7d2dzpkisiuhot1zxtkixgy0pkm029mphlxlkhsq042jb48k65kin4nsrccygjgkmq4d8u51f/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://5i3olmjokc46l5jnqk96sn7maf7wfirfxhswbwxl7cvwaze2p6wravg1w9i39mdc2njnnnv0ezpl1v3lbs75h0i7e6epn90crrwowd9j4tfrz8qbfpy8usbr4boobz95:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 18, "total_num_episodes": 40, "total_success_rate": 0.45, "total_success_rate_lb_0.01": 0.2738732099985334, "total_success_rate_ub_0.01": 0.6341655254095682, "total_success_rate_lb_0.025": 0.306664362588449, "total_success_rate_ub_0.025": 0.614022277272785, "total_success_rate_lb_0.05": 0.3285517841920427, "total_success_rate_ub_0.05": 0.576350063070286, "total_success_rate_lb_0.1": 0.34494671973706914, "total_success_rate_ub_0.1": 0.560576409089371, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "ec522ddf9eb122ad875352661c119e6665bce279dff23c3b347d3e94f5dfebbd", "size": 503, "artifact_path": "wandb-client-artifact://vg8vy0pp1cwd64qun8255cv1fzr91lc83v12pr1v5toc2d1b90tr6b9qdskkz9um2ezhwn344cx8bhqzomx1zmoby6bliodsgogf58a3bek941ssb9a971cs21wy4nnb/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gimkfiwaxk91qzeo7mskixqxxv2ydlltxvzr3zgt2emhl4ufv9pyvs89i5fg16dcdzreyuwystni1agvwav54sdj0eki0oujpvcn5c5i09kpdvmeruf6q5nk7ppt3tnb:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 329}, "eval/total_success_rate": 0.44999998807907104, "eval/total_success_rate_lb_0.01": 0.2738732099533081, "eval/total_success_rate_ub_0.01": 0.6341655254364014, "eval/total_success_rate_lb_0.025": 0.306664377450943, "eval/total_success_rate_ub_0.025": 0.6140222549438477, "eval/total_success_rate_lb_0.05": 0.3285517990589142, "eval/total_success_rate_ub_0.05": 0.5763500332832336, "eval/total_success_rate_lb_0.1": 0.3449467122554779, "eval/total_success_rate_ub_0.1": 0.5605763792991638} \ No newline at end of file diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug-internal.log b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..52ce5ca1f0e8b3dea6a4b5ddc0a3eb1d89e94cc1 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug-internal.log @@ -0,0 +1,853 @@ +2025-01-01 13:21:46,969 INFO StreamThr :1803652 [internal.py:wandb_internal():85] W&B internal server running at pid: 1803652, started at: 2025-01-01 13:21:46.969049 +2025-01-01 13:21:46,971 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:21:46,982 INFO WriterThread:1803652 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb +2025-01-01 13:21:46,983 DEBUG SenderThread:1803652 [sender.py:send():379] send: header +2025-01-01 13:21:46,993 DEBUG SenderThread:1803652 [sender.py:send():379] send: run +2025-01-01 13:21:47,242 INFO SenderThread:1803652 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files +2025-01-01 13:21:47,242 INFO SenderThread:1803652 [sender.py:_start_run_threads():1188] run started: h2q92ngg with start time 1735755706.969843 +2025-01-01 13:21:47,267 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:21:47,267 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:21:47,311 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:21:47,316 DEBUG HandlerThread:1803652 [system_info.py:__init__():26] System info init +2025-01-01 13:21:47,316 DEBUG HandlerThread:1803652 [system_info.py:__init__():41] System info init done +2025-01-01 13:21:47,316 INFO HandlerThread:1803652 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:21:47,316 INFO SystemMonitor:1803652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:21:47,316 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:21:47,317 INFO HandlerThread:1803652 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:21:47,317 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:21:47,317 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:21:47,318 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:21:47,319 INFO SystemMonitor:1803652 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:21:47,328 DEBUG HandlerThread:1803652 [system_info.py:probe():152] Probing system +2025-01-01 13:21:47,330 DEBUG HandlerThread:1803652 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:probe():200] Probing system done +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:21:47.328956', 'startedAt': '2025-01-01T18:21:46.951055', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.79325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.721, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.934, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.022, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 764.9539833068848}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:21:47,335 INFO HandlerThread:1803652 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:21:47,335 INFO HandlerThread:1803652 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:21:47,335 DEBUG HandlerThread:1803652 [system_info.py:_save_code():44] Saving code +2025-01-01 13:21:47,341 DEBUG HandlerThread:1803652 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:21:47,341 DEBUG HandlerThread:1803652 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:21:47,403 DEBUG HandlerThread:1803652 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:21:47,404 INFO HandlerThread:1803652 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:21:47,407 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:21:47,407 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:21:47,407 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:21:47,408 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:21:47,659 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:21:47,659 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:21:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:21:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:21:47,666 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/gi0kv4fc-wandb-metadata.json +2025-01-01 13:21:47,688 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/97com0pf-diff.patch +2025-01-01 13:21:47,694 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/trferv11-code/FISH/eval_robot.py +2025-01-01 13:21:47,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:47,705 INFO HandlerThread:1803652 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/tb" + save: true +} +control { + relay_id: "11441bc7840344528928920deeb35cc2" +} +_info { + stream_id: "h2q92ngg" +} + +2025-01-01 13:21:47,705 DEBUG HandlerThread:1803652 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:21:47,730 DEBUG SenderThread:1803652 [sender.py:send():379] send: telemetry +2025-01-01 13:21:47,754 DEBUG SenderThread:1803652 [sender.py:send():379] send: tbrecord +2025-01-01 13:21:47,755 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:21:47,755 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 with policy live +2025-01-01 13:21:47,951 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/yukgbo6b-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt +2025-01-01 13:21:48,244 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code +2025-01-01 13:21:48,245 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH +2025-01-01 13:21:48,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:49,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:50,243 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:21:50,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:51,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:52,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:52,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:53,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:54,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:55,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:56,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:57,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:21:57,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:58,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:21:59,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:00,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:01,094 DEBUG SenderThread:1803652 [sender.py:send():379] send: config +2025-01-01 13:22:01,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:02,248 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:02,659 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:02,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:02,796 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:03,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:04,249 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:04,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:05,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:06,249 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:06,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:07,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:08,250 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:08,534 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:08,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:09,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:10,251 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:10,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:11,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:12,251 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:12,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:13,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:13,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:14,252 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:14,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:15,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:16,252 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:16,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:17,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:17,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:18,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:18,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:19,253 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:22:19,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:20,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:21,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:22,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:23,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:24,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:24,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:25,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:26,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:27,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:28,256 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:28,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:29,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:29,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:30,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:31,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:32,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:32,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:33,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:34,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:34,810 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:35,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:36,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:37,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:38,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:39,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:40,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:40,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:41,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:42,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:43,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:44,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:45,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:45,667 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:46,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:47,319 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:22:47,320 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:22:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:22:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:22:47,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:48,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:49,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:50,025 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:22:50,025 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 with policy now +2025-01-01 13:22:50,026 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:22:50,027 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:22:50,027 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:22:50,027 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:22:50,261 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media +2025-01-01 13:22:50,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:22:50,295 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/hwy8tvqf-media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,560 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/2fo4ss65-media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:22:50,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:51,033 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:51,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:52,262 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:52,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:53,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:54,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:55,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:56,077 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:22:56,263 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:56,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:57,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:58,264 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:22:58,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:22:59,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:00,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:01,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:01,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:02,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:02,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:03,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:04,670 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:05,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:06,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:06,805 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:07,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:08,267 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:23:08,519 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/hqgrija5-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:23:08,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:09,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:10,268 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:10,671 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:11,936 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:12,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:13,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:14,269 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:14,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:15,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:16,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:17,320 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:23:17,321 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:17,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:17,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:18,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:19,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:20,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:21,672 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:22,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:22,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:23,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:24,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:25,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:26,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:27,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:27,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:28,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:29,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:30,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:31,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:32,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:32,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:32,721 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:33,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:34,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:35,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:36,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:37,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:38,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:38,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:39,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:40,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:41,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:42,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:43,663 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:43,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:44,280 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:44,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:45,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:46,678 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:47,321 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:23:47,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:23:47,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:23:47,676 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:48,049 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:23:48,049 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 with policy now +2025-01-01 13:23:48,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:23:48,053 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:23:48,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:23:48,058 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:48,281 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,282 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:23:48,321 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/wj8s672l-media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,628 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/buvd4j6t-media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:23:48,677 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:49,073 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:49,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:50,282 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:50,686 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:51,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:52,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:53,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:54,283 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:54,668 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:23:54,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:55,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:56,284 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:23:56,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:57,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:58,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:23:59,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:00,405 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:00,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:01,135 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:24:01,136 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:01,136 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:01,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:24:01,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:24:01,679 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:02,286 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:02,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:02,660 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:02,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:03,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:04,287 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:04,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:05,406 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:05,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:06,287 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:06,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:07,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:08,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:09,680 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:10,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:10,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:11,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:12,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:13,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:14,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:15,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:15,681 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:16,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:17,321 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:24:17,660 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:17,661 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:17,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:18,291 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:18,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:19,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:20,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:20,761 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:21,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:22,682 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:23,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:24,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:25,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:26,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:26,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:27,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:28,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:29,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:30,295 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:30,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:31,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,296 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:32,401 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:32,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:32,701 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:33,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:34,297 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:34,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:35,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:36,685 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:37,666 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:37,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:38,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:38,686 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:38,746 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:24:38,746 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 with policy now +2025-01-01 13:24:38,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:24:38,747 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:24:38,748 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:24:38,755 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:24:39,058 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/2vinot8v-media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:24:39,586 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/yzu18ye1-media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:24:39,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:40,298 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:40,693 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:41,692 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:42,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:42,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:43,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:44,300 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:44,698 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:45,721 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:46,691 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:47,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:24:47,661 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:24:47,661 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:24:47,705 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:47,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:48,301 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:24:48,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:49,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:50,696 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:51,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:52,703 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:53,675 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:53,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:54,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:55,698 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:56,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:57,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:58,700 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:24:59,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:24:59,699 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:00,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:01,700 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:02,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:02,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:02,709 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:03,702 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:04,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:04,817 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:05,701 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:06,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:07,704 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:08,708 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:09,712 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:10,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:10,712 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:11,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:12,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:13,707 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:14,709 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:15,677 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:15,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:16,710 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:17,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:25:17,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:17,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:17,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:18,059 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:25:18,059 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:18,059 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:18,311 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:25:18,311 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:25:18,579 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/7das7btf-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:25:18,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:19,711 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:20,312 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:20,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:21,041 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:21,719 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:22,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:23,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:24,313 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:24,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:25,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:26,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:26,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:27,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:28,713 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:29,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:30,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:31,673 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:31,714 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:32,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:32,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:32,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:33,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:34,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:35,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:36,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:36,793 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:37,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:38,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:39,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:40,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:41,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:42,588 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:42,716 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:43,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:44,320 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:44,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:45,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:46,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:47,322 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:25:47,662 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:25:47,662 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:25:47,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:47,800 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:48,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:49,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:50,717 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:51,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:52,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:53,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:53,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:54,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:55,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:56,718 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:57,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:58,055 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:25:58,055 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 with policy now +2025-01-01 13:25:58,058 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:25:58,059 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:25:58,059 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:25:58,059 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:25:58,324 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:25:58,348 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/dv8zver5-media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,627 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/vh1tv1dz-media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:25:58,726 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:25:59,074 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:25:59,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:00,325 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:00,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:01,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:02,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:02,687 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:02,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:03,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:04,327 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:04,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:04,772 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:05,723 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:06,327 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:26:06,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:06,727 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:07,060 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:26:07,061 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:07,082 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:07,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:26:07,732 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:08,328 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:08,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:09,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:09,955 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:10,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:11,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:12,329 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:12,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:13,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:14,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:15,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:15,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:16,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:17,323 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:26:17,665 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:17,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:17,728 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:18,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:19,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:20,332 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:20,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:21,546 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:21,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:22,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:23,729 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:24,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:25,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:26,684 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:26,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:27,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:28,730 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:29,731 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:30,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:31,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:32,304 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:32,336 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:32,664 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:32,665 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:32,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:33,733 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:34,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:35,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:36,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:37,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:37,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:38,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:39,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:40,339 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:40,734 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:41,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:42,683 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:42,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:43,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:44,735 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:45,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:46,305 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:26:46,305 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 with policy now +2025-01-01 13:26:46,306 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:26:46,306 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:26:46,306 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:26:46,308 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:46,341 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:46,342 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval +2025-01-01 13:26:46,590 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/jcss2q0w-media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:46,736 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:46,874 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/zx1veo6m-media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:26:47,323 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:26:47,669 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:26:47,670 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:26:47,739 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:47,779 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:48,342 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:48,743 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:49,746 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:50,744 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:51,747 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:52,343 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:52,751 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:53,248 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:53,750 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:54,344 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:26:54,749 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:55,754 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:56,754 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:57,758 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:58,697 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:26:58,759 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:26:59,756 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:00,756 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:01,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:02,674 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:27:02,675 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:27:02,761 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:03,746 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:03,765 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:04,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:05,762 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:06,762 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:07,772 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:08,765 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:09,694 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:09,768 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:10,349 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:10,592 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/6ttli542-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:10,715 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:27:10,715 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:27:10,766 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:10,928 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:11,165 INFO wandb-upload_0:1803652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp270bkk4f +2025-01-01 13:27:11,318 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:27:11,327 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:11,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media +2025-01-01 13:27:11,351 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval +2025-01-01 13:27:11,351 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table +2025-01-01 13:27:11,530 INFO SenderThread:1803652 [sender.py:send_request_log_artifact():1525] logged artifact run-h2q92ngg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc2NDY0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1MzE1Nw==', 'latestArtifact': None}} +2025-01-01 13:27:11,530 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:27:11,530 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-01 13:27:11,532 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:11,532 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,535 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,535 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:11,535 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,536 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send():379] send: telemetry +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,536 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,536 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,537 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,537 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,537 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,538 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,538 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,538 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,538 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,539 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,539 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,539 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,539 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,540 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,540 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,540 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,541 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,541 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:11,541 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:11,541 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:27:11,717 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/csko8po3-media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:11,763 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:11,780 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/wqj5tiix-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:11,912 INFO wandb-upload_0:1803652 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6o5mgxom +2025-01-01 13:27:12,035 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/8tjeydme-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:12,294 INFO SenderThread:1803652 [sender.py:send_request_log_artifact():1525] logged artifact run-h2q92ngg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc2NDY1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1MzE1OQ==', 'latestArtifact': None}} +2025-01-01 13:27:12,294 DEBUG SenderThread:1803652 [sender.py:send():379] send: files +2025-01-01 13:27:12,294 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json with policy now +2025-01-01 13:27:12,295 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:12,295 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:12,295 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:12,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:12,350 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:12,546 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/mqf1uwvf-media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:12,763 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:13,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:14,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:15,687 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:15,764 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:16,493 DEBUG SenderThread:1803652 [sender.py:send():379] send: exit +2025-01-01 13:27:16,493 INFO SenderThread:1803652 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:27:16,494 INFO SenderThread:1803652 [sender.py:send_exit():588] handling runtime: 329 +2025-01-01 13:27:16,495 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:16,495 INFO SenderThread:1803652 [sender.py:send_exit():594] send defer +2025-01-01 13:27:16,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,495 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:27:16,495 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:27:16,496 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:27:16,496 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:27:16,496 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:27:16,496 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:27:16,496 INFO HandlerThread:1803652 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:27:16,497 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:27:16,497 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:27:16,497 DEBUG SystemMonitor:1803652 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:27:16,497 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:27:16,550 INFO HandlerThread:1803652 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send():379] send: stats +2025-01-01 13:27:16,551 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,551 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:27:16,551 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:27:16,551 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:27:16,551 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:16,551 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:27:17,352 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:18,353 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:23,144 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:27:23,145 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send():379] send: history +2025-01-01 13:27:23,146 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:27:23,146 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,146 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:23,147 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:27:23,147 DEBUG SenderThread:1803652 [sender.py:send():379] send: summary +2025-01-01 13:27:23,151 INFO SenderThread:1803652 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:27:23,151 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,151 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:27:23,152 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:27:23,152 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,152 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:27:23,152 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,152 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:27:23,157 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:27:23,406 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:27:23,406 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:23,406 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:27:23,406 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:23,406 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:27:23,407 INFO Thread-12 :1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:23,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:24,161 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:27:24,161 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:24,162 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,162 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:27:24,162 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,162 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:27:24,162 INFO SenderThread:1803652 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:27:24,164 INFO SenderThread:1803652 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:27:24,166 INFO SenderThread:1803652 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:27:24,183 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:27:24,183 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,183 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:27:24,183 DEBUG SenderThread:1803652 [sender.py:send():379] send: artifact +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:27:24,365 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:27:24,365 INFO SenderThread:1803652 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:27:24,407 INFO SenderThread:1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml config.yaml +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json wandb-summary.json +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:24,408 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt requirements.txt +2025-01-01 13:27:24,411 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log output.log +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/diff.patch diff.patch +2025-01-01 13:27:24,412 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 media/videos/eval/2_eval_2_1c5a41bdc902305f5743.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 media/videos/eval/3_eval_3_2e395aa11844e7b03dc3.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 media/videos/eval/4_eval_4_3f825f4a9faf4b49846d.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 media/videos/eval/0_eval_0_4df3e06ec6ac885c2c7c.mp4 +2025-01-01 13:27:24,415 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 media/videos/eval/1_eval_1_dc0db46e1e3f7c48ac1b.mp4 +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json media/table/eval/total_success_rate_ci_6_ec522ddf9eb122ad8753.table.json +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-01 13:27:24,417 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:27:24,417 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,418 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:27:24,418 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,418 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:27:24,418 INFO SenderThread:1803652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:27:24,495 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:24,495 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:24,603 INFO wandb-upload_2:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/config.yaml +2025-01-01 13:27:24,619 INFO wandb-upload_0:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/wandb-summary.json +2025-01-01 13:27:24,648 INFO wandb-upload_3:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/requirements.txt +2025-01-01 13:27:24,662 INFO wandb-upload_1:1803652 [upload_job.py:push():130] Uploaded file /tmp/tmpxwujc5tqwandb/qxbcjdun-events.out.tfevents.1735755707.leonmkim-ROG-Strix-G15CS-G15CS.1803478.0 +2025-01-01 13:27:24,671 INFO wandb-upload_4:1803652 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/files/output.log +2025-01-01 13:27:24,871 INFO Thread-11 (_thread_body):1803652 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:27:24,871 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,872 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:27:24,872 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,872 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:27:24,872 INFO SenderThread:1803652 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:27:24,873 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:24,873 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:27:24,873 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:27:24,873 INFO SenderThread:1803652 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:27:25,051 INFO SenderThread:1803652 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:27:25,052 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:25,052 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:27:25,052 INFO SenderThread:1803652 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send():379] send: final +2025-01-01 13:27:25,052 DEBUG SenderThread:1803652 [sender.py:send():379] send: footer +2025-01-01 13:27:25,052 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:27:25,053 INFO HandlerThread:1803652 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:27:25,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: defer +2025-01-01 13:27:25,053 INFO SenderThread:1803652 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:27:25,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:25,053 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:27:25,053 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:25,054 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:27:25,054 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:27:25,054 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:27:25,055 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:27:25,057 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:27:25,105 DEBUG SenderThread:1803652 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:27:25,106 INFO MainThread:1803652 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:27:25,107 DEBUG HandlerThread:1803652 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:27:25,107 INFO HandlerThread:1803652 [handler.py:finish():882] shutting down handler +2025-01-01 13:27:26,055 INFO WriterThread:1803652 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [sender.py:finish():1615] shutting down sender +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:27:26,106 INFO SenderThread:1803652 [file_pusher.py:join():175] waiting for file pusher diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug.log b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac36e1365603f790f2d3e0c7a4d5ea6bb16225a0 --- /dev/null +++ b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Configure stats pid to 1803478 +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/settings +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:21:46,958 INFO MainThread:1803478 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug.log +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/logs/debug-internal.log +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:21:46,959 INFO MainThread:1803478 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg'} +2025-01-01 13:21:46,960 INFO MainThread:1803478 [wandb_init.py:init():619] starting backend +2025-01-01 13:21:46,960 INFO MainThread:1803478 [wandb_init.py:init():623] setting up manager +2025-01-01 13:21:46,968 INFO MainThread:1803478 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:21:46,969 INFO MainThread:1803478 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:21:46,982 INFO MainThread:1803478 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:21:46,990 INFO MainThread:1803478 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:21:47,267 INFO MainThread:1803478 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:21:47,306 INFO MainThread:1803478 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:21:47,306 INFO MainThread:1803478 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:21:47,660 INFO MainThread:1803478 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:21:47,662 INFO MainThread:1803478 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:21:47,663 INFO MainThread:1803478 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/h2q92ngg/tb, True +2025-01-01 13:22:01,094 INFO MainThread:1803478 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 13:27:26,107 WARNING MsgRouterThr:1803478 [router.py:message_loop():77] message_loop has been closed diff --git a/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..78e41da84c920114de76f7884bd9236ea14a7262 Binary files /dev/null and b/h2q92ngg/wandb/run-20250101_132146-h2q92ngg/run-h2q92ngg.wandb differ diff --git a/kp7i5uxx/.hydra/config.yaml b/kp7i5uxx/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/kp7i5uxx/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kp7i5uxx/.hydra/hydra.yaml b/kp7i5uxx/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74b6761482c87c3f3d4b5e04a5eb731a4ca317a7 --- /dev/null +++ b/kp7i5uxx/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kp7i5uxx/.hydra/overrides.yaml b/kp7i5uxx/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kp7i5uxx/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kp7i5uxx/episode_rosbags/aligned_depth_to_color_K.npy b/kp7i5uxx/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kp7i5uxx/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kp7i5uxx/episode_rosbags/cam_tf_world.npy b/kp7i5uxx/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kp7i5uxx/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kp7i5uxx/episode_rosbags/color_K.npy b/kp7i5uxx/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kp7i5uxx/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kp7i5uxx/episode_rosbags/depth_K.npy b/kp7i5uxx/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kp7i5uxx/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag b/kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..ff0c9c65813e68024b5a251f767cfdd771c8806f --- /dev/null +++ b/kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8fa0ec7a9d7d534e8e4e464c20ca93547bc4e3eb32494a94badcdccdb80425c +size 1528204095 diff --git a/kp7i5uxx/eval_robot.log b/kp7i5uxx/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..db527465c0c8beaf2167866cb5646118c3e63bd3 --- /dev/null +++ b/kp7i5uxx/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 13:35:33,019][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:35:33,022][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kp7i5uxx/eval_video/0_eval.mp4 b/kp7i5uxx/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0ff7e952ea1237dd423e606a51e11d2e7cc6446 --- /dev/null +++ b/kp7i5uxx/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:568bf5b9d4210e7eaeb5eca28e79bf203e6ae8070667cf4c4111af88fa8ec5f3 +size 1219569 diff --git a/kp7i5uxx/eval_video/1_eval.mp4 b/kp7i5uxx/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d8abfcccadbe6679241cf01c3904a676321500d --- /dev/null +++ b/kp7i5uxx/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:286da0659dcf7249eb987fb6ea583454005bdffef11871bed678d8e9b9213f34 +size 1457426 diff --git a/kp7i5uxx/eval_video/2_eval.mp4 b/kp7i5uxx/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af4907694e8ced4506b60b895634642920d1e3ee --- /dev/null +++ b/kp7i5uxx/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798887158b66dda571d4602a85f1df07f3cc31156408d529f6c1ff6aeae1b95a +size 920526 diff --git a/kp7i5uxx/eval_video/3_eval.mp4 b/kp7i5uxx/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8944db2aca36bc6451561ec16c2225b0477d1424 --- /dev/null +++ b/kp7i5uxx/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:252c2cdff8a70260e6113405ef3b20cd1e595bc667a826e34af66dd3b510946a +size 1471079 diff --git a/kp7i5uxx/eval_video/4_eval.mp4 b/kp7i5uxx/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..916f40be64ddaf89634f2755ea2aeb3680a56fa3 --- /dev/null +++ b/kp7i5uxx/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031275cc1ef6ebf2ca3f28ff6d6f7455a108ed67cc72e495dcd573e22195e541 +size 1497462 diff --git a/kp7i5uxx/tb/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 b/kp7i5uxx/tb/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 new file mode 100644 index 0000000000000000000000000000000000000000..b58ad1483d3f0d47253927835c030239b78599be --- /dev/null +++ b/kp7i5uxx/tb/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed08c1cd9e3de1e8085bae12560a844b36d5c67cce2b92818a43d677b37dd343 +size 6519 diff --git a/kp7i5uxx/wandb/debug-internal.log b/kp7i5uxx/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dda6d4cca0394bb9197103e85c85e164c7643b0a --- /dev/null +++ b/kp7i5uxx/wandb/debug-internal.log @@ -0,0 +1,979 @@ +2025-01-01 13:35:39,054 INFO StreamThr :1805851 [internal.py:wandb_internal():85] W&B internal server running at pid: 1805851, started at: 2025-01-01 13:35:39.053689 +2025-01-01 13:35:39,055 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:35:39,068 INFO WriterThread:1805851 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb +2025-01-01 13:35:39,068 DEBUG SenderThread:1805851 [sender.py:send():379] send: header +2025-01-01 13:35:39,075 DEBUG SenderThread:1805851 [sender.py:send():379] send: run +2025-01-01 13:35:39,289 INFO SenderThread:1805851 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files +2025-01-01 13:35:39,289 INFO SenderThread:1805851 [sender.py:_start_run_threads():1188] run started: kp7i5uxx with start time 1735756539.054218 +2025-01-01 13:35:39,305 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:35:39,305 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:35:39,377 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:35:39,382 DEBUG HandlerThread:1805851 [system_info.py:__init__():26] System info init +2025-01-01 13:35:39,382 DEBUG HandlerThread:1805851 [system_info.py:__init__():41] System info init done +2025-01-01 13:35:39,382 INFO HandlerThread:1805851 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:35:39,382 INFO SystemMonitor:1805851 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:35:39,383 INFO HandlerThread:1805851 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:35:39,383 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:35:39,384 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:35:39,384 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:35:39,385 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:35:39,385 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:35:39,395 DEBUG HandlerThread:1805851 [system_info.py:probe():152] Probing system +2025-01-01 13:35:39,397 DEBUG HandlerThread:1805851 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:35:39,401 DEBUG HandlerThread:1805851 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_info.py:probe():200] Probing system done +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:35:39.395708', 'startedAt': '2025-01-01T18:35:39.043461', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.562, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.62, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.925, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 788.5883674621582}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:35:39,402 INFO HandlerThread:1805851 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:35:39,402 INFO HandlerThread:1805851 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_info.py:_save_code():44] Saving code +2025-01-01 13:35:39,403 DEBUG HandlerThread:1805851 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:35:39,403 DEBUG HandlerThread:1805851 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:35:39,465 DEBUG HandlerThread:1805851 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:35:39,466 INFO HandlerThread:1805851 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:35:39,469 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:35:39,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:39,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:39,711 INFO HandlerThread:1805851 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/tb" + save: true +} +control { + relay_id: "88c7db59d5744c3a9c3f488df34e9e76" +} +_info { + stream_id: "kp7i5uxx" +} + +2025-01-01 13:35:39,711 DEBUG HandlerThread:1805851 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:35:39,726 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/z7atzpws-code/FISH/eval_robot.py +2025-01-01 13:35:39,754 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/dpy4e1d2-wandb-metadata.json +2025-01-01 13:35:39,764 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ynzxf43g-diff.patch +2025-01-01 13:35:39,795 DEBUG SenderThread:1805851 [sender.py:send():379] send: telemetry +2025-01-01 13:35:39,795 DEBUG SenderThread:1805851 [sender.py:send():379] send: tbrecord +2025-01-01 13:35:39,796 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:35:39,796 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 with policy live +2025-01-01 13:35:39,998 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/g62dpeva-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH +2025-01-01 13:35:40,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:41,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:42,290 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:35:42,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:43,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:44,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:44,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:45,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:46,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:47,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:48,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:49,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:49,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:50,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:51,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:52,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:53,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:54,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:54,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:54,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:54,840 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:55,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:56,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:57,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:58,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:59,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:00,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:00,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:01,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:02,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:03,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:04,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:05,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:05,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:06,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:07,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:08,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:09,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:09,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:09,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:10,475 DEBUG SenderThread:1805851 [sender.py:send():379] send: config +2025-01-01 13:36:10,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:11,520 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:11,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:12,298 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:36:12,298 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:12,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:13,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:14,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:15,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:16,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:17,525 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:17,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:18,300 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:18,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:19,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:20,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:21,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:22,301 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:22,629 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:22,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:23,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:24,301 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:24,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:24,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:24,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:25,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:26,302 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:26,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:27,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:27,828 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:28,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:29,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:30,303 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:30,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:31,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:32,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:33,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:33,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:34,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:35,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:36,305 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:36,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:37,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:38,306 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:38,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:39,161 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:39,385 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:36:39,386 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:36:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:39,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:39,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:40,306 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:40,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:41,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:42,307 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:42,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:43,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:44,326 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:44,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:45,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:46,308 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:46,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:47,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:48,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:49,334 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:49,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:50,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:51,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:52,310 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:52,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:53,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:54,310 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:54,337 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:54,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:54,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:54,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:55,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:56,311 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:56,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:57,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:58,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:59,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:59,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:00,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:01,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:02,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:03,722 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:04,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:05,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:05,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:06,722 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:07,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:08,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:09,386 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:37:09,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:09,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:09,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:10,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:10,871 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:11,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:12,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:13,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:14,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:15,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:16,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:16,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:17,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:18,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:19,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:20,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:21,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:21,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:22,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:23,275 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:37:23,275 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 with policy now +2025-01-01 13:37:23,276 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:37:23,277 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:37:23,277 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:37:23,277 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos +2025-01-01 13:37:23,549 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/tgkvvdc1-media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:23,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:23,815 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/dzd5q0x1-media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:24,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:24,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:24,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:25,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:26,726 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:26,821 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:27,726 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:28,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:29,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:30,320 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:30,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:31,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:32,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:32,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:33,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:34,321 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:34,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:35,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:36,322 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:36,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:37,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:38,137 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:38,323 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:38,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:39,387 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:37:39,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:39,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:39,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:40,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:41,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:42,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:43,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:43,797 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:44,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:45,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:46,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:47,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:48,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:49,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:49,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:50,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:51,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:52,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:53,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:54,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:54,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:54,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:54,854 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:55,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:56,328 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:37:56,329 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:56,577 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ph6avbo9-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:37:56,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:57,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:58,329 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:58,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:59,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:00,445 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:00,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:01,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:02,330 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:02,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:03,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:04,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:05,547 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:05,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:06,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:07,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:08,332 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:08,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:09,387 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:38:09,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:09,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:10,333 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:10,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:10,859 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:11,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:12,334 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:12,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:13,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:14,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:15,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:16,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:16,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:17,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:18,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:19,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:20,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:21,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:21,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:22,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:23,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:24,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:24,711 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:25,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:26,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:26,814 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:27,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:28,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:29,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:30,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:31,738 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:32,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:32,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:33,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:34,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:35,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:36,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:37,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:37,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:38,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:39,388 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:38:39,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:39,711 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:39,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:40,343 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:40,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:41,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:42,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:42,800 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:43,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:44,743 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:45,406 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:38:45,406 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 with policy now +2025-01-01 13:38:45,406 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:38:45,407 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:38:45,407 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:38:45,407 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:38:45,677 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/p1br1m95-media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:45,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:38:46,671 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/8006iazq-media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:46,746 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:47,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:47,744 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:48,437 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:48,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:49,746 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:50,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:51,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:52,347 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:52,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:53,481 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:53,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:54,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:54,713 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:54,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:55,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:56,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:57,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:58,349 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:58,581 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:58,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:59,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:00,349 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:00,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:00,604 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/r4jnixte-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:00,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:01,298 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:01,299 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:01,299 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:01,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:01,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:01,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:02,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:02,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:03,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:04,299 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:04,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:05,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:06,352 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:06,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:07,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:08,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:09,388 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:39:09,389 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:09,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:09,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:10,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:11,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:12,354 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:12,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:13,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:14,397 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:14,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:15,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:16,355 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:16,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:17,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:18,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:19,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:19,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:20,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:21,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:22,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:23,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:24,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:24,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:24,770 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:25,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:26,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:27,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:28,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:29,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:30,360 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:30,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:30,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:31,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:32,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:33,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:34,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:35,755 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:35,995 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:36,132 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:39:36,132 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 with policy now +2025-01-01 13:39:36,136 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:39:36,137 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:36,137 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:36,137 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:39:36,395 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/neid7xo6-media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,655 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/mbs2ngmu-media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:37,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:37,755 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:38,363 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:38,760 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:39,389 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:39:39,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:39,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:39,762 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:40,758 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:41,167 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:41,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:42,364 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:42,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:43,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:44,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:45,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:46,268 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:46,765 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:47,767 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:48,366 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:48,769 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:49,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:50,763 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:51,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:52,233 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:52,367 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:52,765 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:53,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:54,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:54,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:54,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:55,069 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:55,069 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:55,069 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:55,368 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:55,368 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:55,769 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:56,369 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:56,767 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:57,772 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:58,070 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:58,771 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:59,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:00,370 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:00,771 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:01,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:02,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:03,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:03,776 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:04,779 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:05,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:06,372 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:06,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:07,784 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:08,373 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:08,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:09,389 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:40:09,389 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:09,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:09,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:10,373 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:10,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:11,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:12,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:13,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:14,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:14,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:15,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:16,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:17,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:18,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:19,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:20,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:20,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:21,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:22,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:23,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:24,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:24,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:24,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:25,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:25,857 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:26,784 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:27,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:28,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:29,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:30,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:31,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:31,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:32,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:33,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:34,787 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:35,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:36,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:36,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:37,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:38,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:39,390 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:40:39,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:39,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:39,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:40,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:41,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:41,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:42,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:43,801 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:44,385 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:44,543 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:40:44,543 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 with policy now +2025-01-01 13:40:44,546 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:40:44,546 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:40:44,547 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:40:44,554 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:40:44,790 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:44,805 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/0cu8wlzn-media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,385 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:40:45,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:40:45,683 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/6fo9emnn-media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,799 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:46,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:46,802 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:47,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:47,798 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:48,800 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:49,802 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:50,387 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:50,805 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:51,806 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:52,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:52,803 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:53,822 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:54,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:54,715 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:54,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:55,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:56,389 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:56,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:57,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:58,390 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:58,405 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:58,804 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:59,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:00,391 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:00,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:01,810 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:02,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:03,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:03,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:04,815 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:05,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:06,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:07,151 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:41:07,152 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:07,152 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:07,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:41:07,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:41:07,647 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/pzxvwm4a-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:41:07,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:08,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:08,815 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:09,200 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:09,390 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:41:09,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:09,718 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:09,820 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:10,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:11,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:12,825 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:13,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:14,249 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:14,395 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:14,829 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:15,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:16,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:17,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:18,397 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:18,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:19,361 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:19,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:20,398 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:20,840 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:21,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:22,399 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:22,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:23,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:24,517 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:24,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:24,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:24,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:25,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:26,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:27,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:28,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:29,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:29,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:30,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:31,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:32,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:33,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:34,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:34,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:35,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:36,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:37,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:38,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:39,391 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:41:39,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:39,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:39,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:39,861 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:40,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:41,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:42,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:43,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:44,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:45,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:45,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:46,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:47,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:48,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:49,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:50,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:51,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:51,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:52,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:53,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:54,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:54,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:54,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:55,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:56,843 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:56,869 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:57,286 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:41:57,286 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 with policy now +2025-01-01 13:41:57,288 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:41:57,289 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:41:57,289 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:57,294 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:41:57,557 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/3poqoqxh-media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,825 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/4n59pftv-media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,848 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:58,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:58,846 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:59,844 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:00,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:01,849 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:02,344 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:02,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:03,851 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:04,414 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:04,856 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:05,853 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:06,849 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:07,443 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:07,850 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:08,416 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:08,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:09,391 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:42:09,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:09,720 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:09,860 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:10,417 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:10,859 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:11,869 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:12,418 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:12,874 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:13,301 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:13,870 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:14,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:15,872 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:16,874 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:17,870 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:18,744 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:18,871 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:19,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:20,882 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:21,421 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:21,681 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/jokw1u7n-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:21,816 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:42:21,816 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:42:21,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:21,987 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:22,304 INFO wandb-upload_2:1805851 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa26cu5t8 +2025-01-01 13:42:22,419 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:42:22,421 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table +2025-01-01 13:42:22,430 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:22,679 INFO SenderThread:1805851 [sender.py:send_request_log_artifact():1525] logged artifact run-kp7i5uxx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3NDUwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NDQ4OA==', 'latestArtifact': None}} +2025-01-01 13:42:22,679 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:42:22,680 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-01 13:42:22,680 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:22,680 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,681 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,681 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send():379] send: telemetry +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,682 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,682 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,683 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,683 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,683 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,683 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,684 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,684 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,685 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,685 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,685 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,685 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,686 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,686 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,686 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,686 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:42:22,875 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/gyv29upa-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:22,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:23,044 INFO wandb-upload_2:1805851 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyhiwzj3l +2025-01-01 13:42:23,153 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/rz829syj-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval +2025-01-01 13:42:23,425 INFO SenderThread:1805851 [sender.py:send_request_log_artifact():1525] logged artifact run-kp7i5uxx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3NDUwNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NDQ5MA==', 'latestArtifact': None}} +2025-01-01 13:42:23,425 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:42:23,425 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json with policy now +2025-01-01 13:42:23,426 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:23,426 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:23,426 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:23,639 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ymwkoltx-media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:23,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:23,890 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/teaxital-media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:24,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:24,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:24,427 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:24,577 DEBUG SenderThread:1805851 [sender.py:send():379] send: exit +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():588] handling runtime: 405 +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():594] send defer +2025-01-01 13:42:24,578 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,578 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:42:24,578 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:42:24,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:42:24,579 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:42:24,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:42:24,579 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:42:24,579 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:42:24,580 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:42:24,580 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:42:24,634 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,634 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:42:24,634 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,634 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:42:25,423 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:26,423 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,252 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:42:31,252 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:31,253 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,253 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:42:31,254 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:31,254 DEBUG SenderThread:1805851 [sender.py:send():379] send: summary +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:31,258 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:42:31,258 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,258 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:42:31,259 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,259 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:42:31,262 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:31,377 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:42:31,377 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,377 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:42:31,377 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,377 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:42:31,425 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:31,425 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:42:31,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:33,541 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:42:33,541 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:33,541 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:33,541 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:42:33,542 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:33,542 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:42:33,542 INFO SenderThread:1805851 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:42:33,544 INFO SenderThread:1805851 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:42:33,548 INFO SenderThread:1805851 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:42:33,569 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:42:33,569 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:33,569 DEBUG SenderThread:1805851 [sender.py:send():379] send: artifact +2025-01-01 13:42:33,569 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:42:33,580 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:42:33,728 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml config.yaml +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json wandb-summary.json +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt requirements.txt +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log output.log +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch diff.patch +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:42:34,433 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:42:34,435 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:34,436 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,436 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:42:34,436 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,436 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:42:34,436 INFO SenderThread:1805851 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:42:34,580 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:34,581 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:34,616 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:42:34,620 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:34,686 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt +2025-01-01 13:42:34,698 INFO wandb-upload_3:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/6avruo3v-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:34,699 INFO wandb-upload_4:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:34,899 INFO Thread-11 (_thread_body):1805851 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:42:34,899 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,899 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:42:34,900 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,900 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:42:34,900 INFO SenderThread:1805851 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:42:34,901 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:42:34,901 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,901 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:42:34,902 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,902 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:42:34,902 INFO SenderThread:1805851 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:42:35,079 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:35,079 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:42:35,079 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:42:35,080 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:42:35,080 DEBUG SenderThread:1805851 [sender.py:send():379] send: final +2025-01-01 13:42:35,080 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:35,080 DEBUG SenderThread:1805851 [sender.py:send():379] send: footer +2025-01-01 13:42:35,080 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:42:35,081 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:35,081 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:42:35,081 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:35,081 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:35,082 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:42:35,082 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:35,083 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:42:35,086 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:42:35,086 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:42:35,087 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:35,139 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:42:35,140 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:42:35,140 INFO HandlerThread:1805851 [handler.py:finish():882] shutting down handler +2025-01-01 13:42:36,082 INFO WriterThread:1805851 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb +2025-01-01 13:42:36,139 INFO SenderThread:1805851 [sender.py:finish():1615] shutting down sender +2025-01-01 13:42:36,140 INFO SenderThread:1805851 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:42:36,140 INFO SenderThread:1805851 [file_pusher.py:join():175] waiting for file pusher diff --git a/kp7i5uxx/wandb/debug.log b/kp7i5uxx/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d34b7f3abaf8f32719fd44ab574d2008ee277c86 --- /dev/null +++ b/kp7i5uxx/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:35:39,044 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Configure stats pid to 1805676 +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/settings +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug.log +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug-internal.log +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx'} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():619] starting backend +2025-01-01 13:35:39,046 INFO MainThread:1805676 [wandb_init.py:init():623] setting up manager +2025-01-01 13:35:39,052 INFO MainThread:1805676 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:35:39,053 INFO MainThread:1805676 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:35:39,067 INFO MainThread:1805676 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:35:39,073 INFO MainThread:1805676 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:35:39,305 INFO MainThread:1805676 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:35:39,372 INFO MainThread:1805676 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:35:39,372 INFO MainThread:1805676 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:35:39,710 INFO MainThread:1805676 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:35:39,711 INFO MainThread:1805676 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/tb, True +2025-01-01 13:36:10,475 INFO MainThread:1805676 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 13:42:36,141 WARNING MsgRouterThr:1805676 [router.py:message_loop():77] message_loop has been closed diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2a3ef5b87d4f042c3f7e81863dba9fe2495687 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735756539 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f9cbfaf74a18317e7403f01a665bc5e3d98239c --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..93b4606f3b367d50590c1fcc1ef2d3831e2bfe78 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[19, 50, 0.38, 0.23051038389827355, 0.5467739999200733, 0.2575774640329739, 0.5273207351510719, 0.27605911348768597, 0.49354415759573356, 0.29055374119894417, 0.47815738991339773]]} \ No newline at end of file diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0ff7e952ea1237dd423e606a51e11d2e7cc6446 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:568bf5b9d4210e7eaeb5eca28e79bf203e6ae8070667cf4c4111af88fa8ec5f3 +size 1219569 diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d8abfcccadbe6679241cf01c3904a676321500d --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:286da0659dcf7249eb987fb6ea583454005bdffef11871bed678d8e9b9213f34 +size 1457426 diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af4907694e8ced4506b60b895634642920d1e3ee --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798887158b66dda571d4602a85f1df07f3cc31156408d529f6c1ff6aeae1b95a +size 920526 diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8944db2aca36bc6451561ec16c2225b0477d1424 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:252c2cdff8a70260e6113405ef3b20cd1e595bc667a826e34af66dd3b510946a +size 1471079 diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..916f40be64ddaf89634f2755ea2aeb3680a56fa3 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031275cc1ef6ebf2ca3f28ff6d6f7455a108ed67cc72e495dcd573e22195e541 +size 1497462 diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9d857f30d76b3d0ebef85855beee719b48b5473e --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log @@ -0,0 +1,296 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735756570.476406]: resetting environment +[INFO] [1735756570.480472]: cleared current plan +[INFO] [1735756570.481572]: moving to home +Move to target timeout: 5.00064492225647. Trans error: 0.006144320126622915. Rot error: 0.6988024159382534 +Move to target timeout: 5.0997583866119385. Trans error: 0.0062260583843804975. Rot error: 1.4973937811502784 +[INFO] [1735756581.584540]: reached home +[INFO] [1735756581.586568]: reset action history +[INFO] [1735756584.296423]: environment reset +Starting episode 0 +[INFO] [1735756584.300790]: resetting environment +[INFO] [1735756584.305450]: cleared current plan +[INFO] [1735756584.306394]: moving to home +Move to target timeout: 5.00069522857666. Trans error: 0.00594179006293416. Rot error: 0.6346190699799652 +Move to target timeout: 5.100041151046753. Trans error: 0.006283691882536829. Rot error: 1.569056053370459 +[INFO] [1735756595.409278]: reached home +[INFO] [1735756595.410717]: reset action history +[INFO] [1735756598.120280]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735756600.283245]: resetting environment +[INFO] [1735756600.287847]: cleared current plan +[INFO] [1735756600.288742]: moving to home +Move to target timeout: 5.000744342803955. Trans error: 0.005901044234633446. Rot error: 0.6303309455306939 +Move to target timeout: 5.099611520767212. Trans error: 0.00627635166067809. Rot error: 1.5709272737148106 +[INFO] [1735756611.391735]: reached home +[INFO] [1735756611.395361]: reset action history +[INFO] [1735756614.103195]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/episode_rosbags/episode_0_2025-01-01-13-36-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1805676.log +started roslaunch server http://158.130.50.37:34273/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1805960] +process[print_text-2]: started with pid [1805984] +process[rosbag_record-3]: started with pid [1805985] +started bagging! +For topic gripper_width: timestamp difference is 113553449 for nearest: 1735756615252770110 - target: 1735756615139216661 at idx 20 +preempting episode +[INFO] [1735756642.236751]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735756643.044605]: Stored episode 1. +[INFO] [1735756643.045241]: Saving video... +[INFO] [1735756643.259064]: Video saved! +Starting episode 1 +[INFO] [1735756643.277085]: resetting environment +[INFO] [1735756643.282392]: cleared current plan +[INFO] [1735756643.282667]: moving to home +Move to target timeout: 5.001591205596924. Trans error: 0.008573123253881931. Rot error: 1.345069291308115 +Move to target timeout: 5.099532604217529. Trans error: 0.006311093925642377. Rot error: 1.0935369078658832 +[INFO] [1735756654.387036]: reached home +[INFO] [1735756654.387532]: reset action history +[INFO] [1735756657.096798]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735756675.438680]: resetting environment +[INFO] [1735756675.443279]: cleared current plan +[INFO] [1735756675.443491]: moving to home +Move to target timeout: 5.000792503356934. Trans error: 0.006043861620128155. Rot error: 0.6899985176052081 +Move to target timeout: 5.099772691726685. Trans error: 0.006186896566894518. Rot error: 1.477274499466747 +[INFO] [1735756686.545266]: reached home +[INFO] [1735756686.545676]: reset action history +[INFO] [1735756689.253523]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/episode_rosbags/episode_1_2025-01-01-13-38-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1805676.log +started roslaunch server http://158.130.50.37:34219/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1806111] +process[print_text-5]: started with pid [1806112] +process[rosbag_record-6]: started with pid [1806136] +started bagging! +For topic gripper_width: timestamp difference is -102750679 for nearest: 1735756718586144308 - target: 1735756718688894987 at idx 249 +[INFO] [1735756723.965091]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735756724.713536]: Stored episode 2. +[INFO] [1735756724.713794]: Saving video... +[INFO] [1735756725.385727]: Video saved! +Starting episode 2 +[INFO] [1735756725.408638]: resetting environment +[INFO] [1735756725.436301]: cleared current plan +[INFO] [1735756725.436601]: moving to home +[WARN] [1735756725.309988]: Plan exhausted +[WARN] [1735756725.350262]: Plan exhausted +[WARN] [1735756725.389670]: Plan exhausted +[WARN] [1735756725.429719]: Plan exhausted +Move to target timeout: 5.001381874084473. Trans error: 0.006066302768886089. Rot error: 0.7562746873051365 +Move to target timeout: 5.095080614089966. Trans error: 0.006348715599589563. Rot error: 1.022326354217457 +[INFO] [1735756736.539555]: reached home +[INFO] [1735756736.540454]: reset action history +[INFO] [1735756739.320280]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735756740.345078]: resetting environment +[INFO] [1735756740.364695]: cleared current plan +[INFO] [1735756740.364959]: moving to home +Move to target timeout: 5.000652551651001. Trans error: 0.006193224806338549. Rot error: 0.6871897091776452 +Move to target timeout: 5.099435806274414. Trans error: 0.00613790406368913. Rot error: 1.4307635848887417 +[INFO] [1735756751.469216]: reached home +[INFO] [1735756751.469714]: reset action history +[INFO] [1735756754.179301]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1805676.log +started roslaunch server http://158.130.50.37:36081/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1806258] +process[print_text-8]: started with pid [1806259] +process[rosbag_record-9]: started with pid [1806283] +started bagging! +For topic gripper_width: timestamp difference is 104361026 for nearest: 1735756768952811925 - target: 1735756768848450899 at idx 213 +preempting episode +[INFO] [1735756774.861027]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit[pub_text-7] killing on exit +[WARN] [1735756775.994805]: Plan exhausted +[WARN] [1735756776.031971]: Plan exhausted +[WARN] [1735756776.069933]: Plan exhausted +[WARN] [1735756776.110263]: Plan exhausted +[WARN] [1735756776.149764]: Plan exhausted +[INFO] [1735756775.601228]: Stored episode 3. +[INFO] [1735756775.602558]: Saving video... +[INFO] [1735756776.113591]: Video saved! +Starting episode 3 +[INFO] [1735756776.139046]: resetting environment +[INFO] [1735756776.160878]: cleared current plan +[INFO] [1735756776.162604]: moving to home +Move to target timeout: 5.001071214675903. Trans error: 0.008043854497373104. Rot error: 1.1629706514449176 +Move to target timeout: 5.0961525440216064. Trans error: 0.006336522353988573. Rot error: 1.0652879045811474 +[INFO] [1735756787.274906]: reached home +[INFO] [1735756787.278054]: reset action history +[INFO] [1735756790.189442]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735756794.434576]: resetting environment +[INFO] [1735756794.471200]: cleared current plan +[INFO] [1735756794.471413]: moving to home +Move to target timeout: 5.0028276443481445. Trans error: 0.006188113708049059. Rot error: 0.707023765814245 +Move to target timeout: 5.095744371414185. Trans error: 0.00618777876308147. Rot error: 1.4798608763748842 +[INFO] [1735756805.574949]: reached home +[INFO] [1735756805.577726]: reset action history +[INFO] [1735756808.356963]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/episode_rosbags/episode_3_2025-01-01-13-40-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1805676.log +started roslaunch server http://158.130.50.37:42265/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1806401] +process[print_text-11]: started with pid [1806402] +process[rosbag_record-12]: started with pid [1806426] +started bagging! +[INFO] [1735756843.154814]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[WARN] [1735756844.520067]: Plan exhausted +[WARN] [1735756844.568516]: Plan exhausted +[INFO] [1735756843.947691]: Stored episode 4. +[INFO] [1735756843.950365]: Saving video... +[INFO] [1735756844.498912]: Video saved! +Starting episode 4 +[INFO] [1735756844.558473]: resetting environment +[INFO] [1735756844.586952]: cleared current plan +[INFO] [1735756844.593260]: moving to home +Move to target timeout: 5.000797510147095. Trans error: 0.008674994111061096. Rot error: 1.3754925502428135 +Move to target timeout: 5.003183841705322. Trans error: 0.00630896572620736. Rot error: 1.0599804440333778 +[INFO] [1735756855.609690]: reached home +[INFO] [1735756855.610691]: reset action history +[INFO] [1735756858.402194]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735756867.167199]: resetting environment +[INFO] [1735756867.197185]: cleared current plan +[INFO] [1735756867.199800]: moving to home +Move to target timeout: 5.001055717468262. Trans error: 0.0061719692312181. Rot error: 0.7014701748400338 +Move to target timeout: 5.095877170562744. Trans error: 0.006176365958566894. Rot error: 1.478844569457587 +[INFO] [1735756878.315767]: reached home +[INFO] [1735756878.321993]: reset action history +[INFO] [1735756881.110797]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/episode_rosbags/episode_4_2025-01-01-13-41-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1805676.log +started roslaunch server http://158.130.50.37:42299/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1806565] +process[print_text-14]: started with pid [1806566] +process[rosbag_record-15]: started with pid [1806567] +started bagging! +[INFO] [1735756916.054239]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735756916.868361]: Stored episode 5. +[INFO] [1735756916.873459]: Saving video... +[INFO] [1735756917.223165]: Video saved! +[INFO] [1735756917.288650]: resetting environment +[INFO] [1735756917.334307]: cleared current plan +[INFO] [1735756917.337160]: moving to home +Move to target timeout: 5.001713752746582. Trans error: 0.00849151611328125. Rot error: 1.3023297314205666 +Move to target timeout: 5.094778060913086. Trans error: 0.006316821996142619. Rot error: 1.0713396883086825 +[INFO] [1735756928.446658]: reached home +[INFO] [1735756928.449417]: reset action history +[INFO] [1735756931.257005]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..101aa84c8c30fa7ffa1bb381b2d66f5c37ab5cef --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T18:35:39.395708", + "startedAt": "2025-01-01T18:35:39.043461", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.562, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.62, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.925, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 788.5883674621582 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d6a87c2aee2064177f0b1b6cc3252ffa590acbfd --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "568bf5b9d4210e7eaeb5eca28e79bf203e6ae8070667cf4c4111af88fa8ec5f3", "size": 1219569, "path": "media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4"}, "global_step": 584, "_timestamp": 1735756942.3179502, "_runtime": 403.2637321949005, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "286da0659dcf7249eb987fb6ea583454005bdffef11871bed678d8e9b9213f34", "size": 1457426, "path": "media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4"}, "eval/num_success": 1.0, "episode": 50.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "798887158b66dda571d4602a85f1df07f3cc31156408d529f6c1ff6aeae1b95a", "size": 920526, "path": "media/videos/eval/2_eval_2_798887158b66dda571d4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "252c2cdff8a70260e6113405ef3b20cd1e595bc667a826e34af66dd3b510946a", "size": 1471079, "path": "media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "031275cc1ef6ebf2ca3f28ff6d6f7455a108ed67cc72e495dcd573e22195e541", "size": 1497462, "path": "media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://fbey4vujtt0gqvwomo4bfifeuiqwhto6ryx06ajzgn7sn9bk7augly8aeonkgsq42zu992nlbt31wp1u9ayo66pgsgf7v1ntdnioi5h0k8frmt1ud09vt4va5v7l2ors/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://g8guqjj0h93cotx7odk1wh8anq39ugs6ab7htvn2cf215vr8pgwfd633eem8mva4y6ummgorgh7he80gu90jzgcyjfdi7ypu4slh2s56h99tpd56hhsrngpkiwx0k2r1:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 19, "total_num_episodes": 50, "total_success_rate": 0.38, "total_success_rate_lb_0.01": 0.23051038389827355, "total_success_rate_ub_0.01": 0.5467739999200733, "total_success_rate_lb_0.025": 0.2575774640329739, "total_success_rate_ub_0.025": 0.5273207351510719, "total_success_rate_lb_0.05": 0.27605911348768597, "total_success_rate_ub_0.05": 0.49354415759573356, "total_success_rate_lb_0.1": 0.29055374119894417, "total_success_rate_ub_0.1": 0.47815738991339773, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "a19350b7b1a9e28e296de0c3cbd54c95d8712b32b1650082f42904909be1968c", "size": 511, "artifact_path": "wandb-client-artifact://y4268agnvjk17ghk9mak8k12qqzna74pbeg3pj30leuwpmdaca46qgabkscw5muvwjfxcfuztwbj8zoiuzamuqkxkaja930dyfuzchdp8h7wpzjxmzsugv4c1j1boyzi/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://k87mhnup8vv4d4l75ufl34xc8rew3npbddcu2gtsxfn3tvsqgxb8axcs11548xrxbpfm3y1vj0rpevxazlx6rru62gfcswlwjjjssysmp4lxtgz5hkoawn4ahgqo4ea9:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 405}, "eval/total_success_rate": 0.3799999952316284, "eval/total_success_rate_lb_0.01": 0.2305103838443756, "eval/total_success_rate_ub_0.01": 0.5467739701271057, "eval/total_success_rate_lb_0.025": 0.25757747888565063, "eval/total_success_rate_ub_0.025": 0.5273207426071167, "eval/total_success_rate_lb_0.05": 0.2760591208934784, "eval/total_success_rate_ub_0.05": 0.49354416131973267, "eval/total_success_rate_lb_0.1": 0.2905537486076355, "eval/total_success_rate_ub_0.1": 0.4781574010848999} \ No newline at end of file diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug-internal.log b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dda6d4cca0394bb9197103e85c85e164c7643b0a --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug-internal.log @@ -0,0 +1,979 @@ +2025-01-01 13:35:39,054 INFO StreamThr :1805851 [internal.py:wandb_internal():85] W&B internal server running at pid: 1805851, started at: 2025-01-01 13:35:39.053689 +2025-01-01 13:35:39,055 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:35:39,068 INFO WriterThread:1805851 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb +2025-01-01 13:35:39,068 DEBUG SenderThread:1805851 [sender.py:send():379] send: header +2025-01-01 13:35:39,075 DEBUG SenderThread:1805851 [sender.py:send():379] send: run +2025-01-01 13:35:39,289 INFO SenderThread:1805851 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files +2025-01-01 13:35:39,289 INFO SenderThread:1805851 [sender.py:_start_run_threads():1188] run started: kp7i5uxx with start time 1735756539.054218 +2025-01-01 13:35:39,305 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:35:39,305 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:35:39,377 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:35:39,382 DEBUG HandlerThread:1805851 [system_info.py:__init__():26] System info init +2025-01-01 13:35:39,382 DEBUG HandlerThread:1805851 [system_info.py:__init__():41] System info init done +2025-01-01 13:35:39,382 INFO HandlerThread:1805851 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:35:39,382 INFO SystemMonitor:1805851 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:35:39,383 INFO HandlerThread:1805851 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:35:39,383 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:35:39,384 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:35:39,384 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:35:39,385 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:35:39,385 INFO SystemMonitor:1805851 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:35:39,395 DEBUG HandlerThread:1805851 [system_info.py:probe():152] Probing system +2025-01-01 13:35:39,397 DEBUG HandlerThread:1805851 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:35:39,401 DEBUG HandlerThread:1805851 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_info.py:probe():200] Probing system done +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:35:39.395708', 'startedAt': '2025-01-01T18:35:39.043461', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.562, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.62, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.925, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 788.5883674621582}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:35:39,402 INFO HandlerThread:1805851 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:35:39,402 INFO HandlerThread:1805851 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:35:39,402 DEBUG HandlerThread:1805851 [system_info.py:_save_code():44] Saving code +2025-01-01 13:35:39,403 DEBUG HandlerThread:1805851 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:35:39,403 DEBUG HandlerThread:1805851 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:35:39,465 DEBUG HandlerThread:1805851 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:35:39,466 INFO HandlerThread:1805851 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:35:39,469 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:35:39,469 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:35:39,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:39,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:39,711 INFO HandlerThread:1805851 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/tb" + save: true +} +control { + relay_id: "88c7db59d5744c3a9c3f488df34e9e76" +} +_info { + stream_id: "kp7i5uxx" +} + +2025-01-01 13:35:39,711 DEBUG HandlerThread:1805851 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:35:39,726 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/z7atzpws-code/FISH/eval_robot.py +2025-01-01 13:35:39,754 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/dpy4e1d2-wandb-metadata.json +2025-01-01 13:35:39,764 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ynzxf43g-diff.patch +2025-01-01 13:35:39,795 DEBUG SenderThread:1805851 [sender.py:send():379] send: telemetry +2025-01-01 13:35:39,795 DEBUG SenderThread:1805851 [sender.py:send():379] send: tbrecord +2025-01-01 13:35:39,796 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:35:39,796 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 with policy live +2025-01-01 13:35:39,998 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/g62dpeva-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code +2025-01-01 13:35:40,291 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH +2025-01-01 13:35:40,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:41,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:42,290 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:35:42,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:43,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:44,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:44,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:45,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:46,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:47,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:48,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:49,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:49,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:50,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:51,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:52,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:53,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:54,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:35:54,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:35:54,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:54,840 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:35:55,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:56,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:57,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:58,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:35:59,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:00,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:00,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:01,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:02,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:03,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:04,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:05,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:05,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:06,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:07,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:08,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:09,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:09,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:09,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:10,475 DEBUG SenderThread:1805851 [sender.py:send():379] send: config +2025-01-01 13:36:10,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:11,520 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:11,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:12,298 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:36:12,298 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:12,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:13,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:14,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:15,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:16,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:17,525 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:17,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:18,300 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:18,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:19,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:20,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:21,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:22,301 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:22,629 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:22,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:23,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:24,301 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:24,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:24,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:24,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:25,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:26,302 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:26,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:27,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:27,828 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:28,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:29,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:30,303 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:30,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:31,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:32,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:33,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:33,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:34,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:35,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:36,305 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:36,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:37,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:38,306 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:38,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:39,161 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:39,385 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:36:39,386 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:36:39,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:39,709 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:39,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:40,306 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:40,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:41,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:42,307 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:42,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:43,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:44,326 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:44,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:45,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:46,308 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:46,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:47,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:48,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:49,334 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:49,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:50,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:51,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:52,310 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:52,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:53,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:54,310 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:54,337 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:54,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:36:54,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:36:54,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:55,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:56,311 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:36:56,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:57,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:58,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:36:59,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:36:59,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:00,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:01,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:02,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:03,722 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:04,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:05,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:05,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:06,722 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:07,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:08,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:09,386 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:37:09,709 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:09,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:09,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:10,723 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:10,871 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:11,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:12,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:13,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:14,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:15,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:16,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:16,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:17,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:18,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:19,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:20,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:21,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:21,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:22,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:23,275 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:37:23,275 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 with policy now +2025-01-01 13:37:23,276 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:37:23,277 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:37:23,277 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:37:23,277 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:37:23,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos +2025-01-01 13:37:23,549 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/tgkvvdc1-media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:23,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:23,815 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/dzd5q0x1-media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:37:24,318 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:24,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:24,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:25,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:26,726 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:26,821 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:27,726 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:28,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:29,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:30,320 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:30,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:31,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:32,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:32,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:33,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:34,321 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:34,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:35,728 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:36,322 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:36,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:37,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:38,137 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:38,323 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:38,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:39,387 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:37:39,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:39,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:39,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:40,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:41,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:42,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:43,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:43,797 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:44,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:45,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:46,729 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:47,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:48,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:49,716 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:49,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:50,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:51,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:52,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:53,730 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:54,710 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:37:54,710 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:37:54,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:54,854 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:37:55,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:56,328 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:37:56,329 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:56,577 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ph6avbo9-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:37:56,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:57,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:58,329 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:37:58,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:37:59,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:00,445 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:00,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:01,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:02,330 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:02,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:03,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:04,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:05,547 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:05,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:06,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:07,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:08,332 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:08,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:09,387 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:38:09,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:09,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:10,333 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:10,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:10,859 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:11,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:12,334 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:12,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:13,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:14,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:15,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:16,717 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:16,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:17,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:18,734 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:19,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:20,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:21,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:21,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:22,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:23,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:24,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:24,711 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:25,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:26,736 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:26,814 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:27,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:28,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:29,737 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:30,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:31,738 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:32,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:32,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:33,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:34,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:35,739 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:36,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:37,719 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:37,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:38,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:39,388 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:38:39,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:39,711 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:39,740 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:40,343 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:40,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:41,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:42,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:42,800 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:43,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:44,743 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:45,406 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:38:45,406 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 with policy now +2025-01-01 13:38:45,406 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:38:45,407 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:38:45,407 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:38:45,407 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:38:45,677 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/p1br1m95-media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:45,741 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:46,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:38:46,671 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/8006iazq-media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:38:46,746 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:47,345 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:47,744 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:48,437 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:48,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:49,746 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:50,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:51,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:52,347 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:52,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:53,481 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:53,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:54,713 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:38:54,713 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:38:54,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:55,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:56,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:57,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:58,349 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:38:58,581 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:38:58,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:38:59,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:00,349 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:00,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:00,604 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/r4jnixte-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:00,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:01,298 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:01,299 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:01,299 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:01,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:01,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:01,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:02,350 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:02,747 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:03,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:04,299 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:04,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:05,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:06,352 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:06,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:07,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:08,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:09,388 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:39:09,389 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:09,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:09,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:10,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:11,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:12,354 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:12,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:13,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:14,397 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:14,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:15,748 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:16,355 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:16,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:17,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:18,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:19,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:19,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:20,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:21,749 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:22,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:23,750 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:24,711 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:24,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:24,753 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:24,770 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:25,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:26,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:27,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:28,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:29,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:30,360 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:30,721 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:30,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:31,751 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:32,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:33,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:34,752 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:35,755 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:35,995 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:36,132 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:39:36,132 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 with policy now +2025-01-01 13:39:36,136 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:39:36,137 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:36,137 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:36,137 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:39:36,395 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/neid7xo6-media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,655 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/mbs2ngmu-media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:39:36,757 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:37,362 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:37,755 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:38,363 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:38,760 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:39,389 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:39:39,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:39,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:39,762 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:40,758 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:41,167 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:41,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:42,364 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:42,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:43,761 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:44,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:45,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:46,268 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:46,765 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:47,767 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:48,366 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:48,769 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:49,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:50,763 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:51,766 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:52,233 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:52,367 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:52,765 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:53,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:54,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:39:54,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:39:54,764 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:55,069 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:39:55,069 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:39:55,069 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:39:55,368 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:39:55,368 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:39:55,769 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:56,369 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:39:56,767 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:57,772 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:58,070 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:39:58,771 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:39:59,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:00,370 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:00,771 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:01,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:02,773 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:03,727 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:03,776 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:04,779 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:05,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:06,372 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:06,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:07,784 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:08,373 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:08,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:09,389 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:40:09,389 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:09,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:09,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:09,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:10,373 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:10,780 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:11,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:12,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:13,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:14,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:14,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:15,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:16,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:17,781 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:18,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:19,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:20,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:20,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:21,782 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:22,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:23,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:24,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:24,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:24,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:25,783 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:25,857 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:26,784 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:27,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:28,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:29,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:30,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:31,724 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:31,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:32,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:33,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:34,787 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:35,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:36,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:36,785 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:37,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:38,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:39,390 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:40:39,712 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:39,712 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:39,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:40,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:41,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:41,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:42,786 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:43,801 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:44,385 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:44,543 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:40:44,543 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 with policy now +2025-01-01 13:40:44,546 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:40:44,546 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:40:44,547 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:40:44,554 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:40:44,790 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:44,805 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/0cu8wlzn-media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,385 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:40:45,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:40:45,683 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/6fo9emnn-media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:40:45,799 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:46,386 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:46,802 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:47,725 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:47,798 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:48,800 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:49,802 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:50,387 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:50,805 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:51,806 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:52,732 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:52,803 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:53,822 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:54,715 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:40:54,715 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:40:54,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:55,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:56,389 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:56,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:57,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:58,390 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:40:58,405 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:40:58,804 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:40:59,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:00,391 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:00,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:01,810 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:02,809 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:03,733 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:03,808 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:04,815 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:05,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:06,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:07,151 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:41:07,152 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:07,152 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:07,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:41:07,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:41:07,647 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/pzxvwm4a-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:41:07,811 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:08,393 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:08,815 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:09,200 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:09,390 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:41:09,718 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:09,718 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:09,820 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:10,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:11,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:12,825 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:13,826 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:14,249 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:14,395 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:14,829 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:15,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:16,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:17,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:18,397 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:18,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:19,361 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:19,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:20,398 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:20,840 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:21,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:22,399 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:22,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:23,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:24,517 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:24,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:24,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:24,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:25,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:26,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:27,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:28,831 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:29,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:29,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:30,832 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:31,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:32,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:33,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:34,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:34,833 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:35,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:36,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:37,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:38,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:39,391 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:41:39,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:39,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:39,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:39,861 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:40,834 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:41,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:42,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:43,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:44,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:45,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:45,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:46,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:47,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:48,835 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:49,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:50,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:51,731 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:51,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:52,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:53,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:54,714 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:41:54,714 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:41:54,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:55,836 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:56,843 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:56,869 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:41:57,286 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:41:57,286 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 with policy now +2025-01-01 13:41:57,288 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:41:57,289 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:41:57,289 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:41:57,294 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval +2025-01-01 13:41:57,557 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/3poqoqxh-media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,825 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/4n59pftv-media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:41:57,848 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:58,412 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:41:58,846 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:41:59,844 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:00,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:01,849 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:02,344 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:02,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:03,851 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:04,414 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:04,856 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:05,853 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:06,849 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:07,443 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:07,850 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:08,416 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:08,858 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:09,391 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:42:09,720 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:42:09,720 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:42:09,860 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:10,417 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:10,859 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:11,869 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:12,418 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:12,874 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:13,301 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:13,870 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:14,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:15,872 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:16,874 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:17,870 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:18,744 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:18,871 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:19,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:20,882 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:21,421 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:21,681 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/jokw1u7n-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:21,816 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:42:21,816 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:42:21,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:21,987 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:22,304 INFO wandb-upload_2:1805851 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa26cu5t8 +2025-01-01 13:42:22,419 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:42:22,421 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval +2025-01-01 13:42:22,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table +2025-01-01 13:42:22,430 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:42:22,679 INFO SenderThread:1805851 [sender.py:send_request_log_artifact():1525] logged artifact run-kp7i5uxx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3NDUwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NDQ4OA==', 'latestArtifact': None}} +2025-01-01 13:42:22,679 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:42:22,680 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-01 13:42:22,680 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:22,680 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,681 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,681 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send():379] send: telemetry +2025-01-01 13:42:22,681 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,682 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,682 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,682 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,683 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,683 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,683 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,683 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,684 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,684 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,684 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,685 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,685 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,685 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,685 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,686 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,686 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:22,686 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:22,686 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:42:22,875 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/gyv29upa-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:22,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:23,044 INFO wandb-upload_2:1805851 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyhiwzj3l +2025-01-01 13:42:23,153 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/rz829syj-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:23,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval +2025-01-01 13:42:23,425 INFO SenderThread:1805851 [sender.py:send_request_log_artifact():1525] logged artifact run-kp7i5uxx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc3NDUwNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NDQ5MA==', 'latestArtifact': None}} +2025-01-01 13:42:23,425 DEBUG SenderThread:1805851 [sender.py:send():379] send: files +2025-01-01 13:42:23,425 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json with policy now +2025-01-01 13:42:23,426 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:23,426 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:23,426 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:23,639 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/ymwkoltx-media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:23,875 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:23,890 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/teaxital-media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:24,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:24,422 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:24,427 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:24,577 DEBUG SenderThread:1805851 [sender.py:send():379] send: exit +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():588] handling runtime: 405 +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:24,578 INFO SenderThread:1805851 [sender.py:send_exit():594] send defer +2025-01-01 13:42:24,578 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,578 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:42:24,578 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:42:24,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:42:24,579 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:42:24,579 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:42:24,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:42:24,579 INFO HandlerThread:1805851 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:42:24,579 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:42:24,579 DEBUG SystemMonitor:1805851 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:42:24,580 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:42:24,580 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:42:24,633 INFO HandlerThread:1805851 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send():379] send: stats +2025-01-01 13:42:24,634 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,634 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:42:24,634 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:42:24,634 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:42:24,634 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:24,634 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:42:25,423 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:26,423 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:31,251 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,252 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:42:31,252 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send():379] send: history +2025-01-01 13:42:31,252 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:42:31,253 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,253 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:42:31,254 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:31,254 DEBUG SenderThread:1805851 [sender.py:send():379] send: summary +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:42:31,258 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:42:31,258 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:42:31,258 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,258 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:42:31,259 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,259 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:42:31,262 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:42:31,377 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:42:31,377 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:31,377 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:42:31,377 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:31,377 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:42:31,425 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:31,425 INFO Thread-12 :1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:42:31,579 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:33,541 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:42:33,541 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:33,541 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:33,541 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:42:33,542 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:33,542 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:42:33,542 INFO SenderThread:1805851 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:42:33,544 INFO SenderThread:1805851 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:42:33,548 INFO SenderThread:1805851 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:42:33,569 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:42:33,569 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:33,569 DEBUG SenderThread:1805851 [sender.py:send():379] send: artifact +2025-01-01 13:42:33,569 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:42:33,580 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:42:33,728 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:42:33,728 INFO SenderThread:1805851 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files +2025-01-01 13:42:34,426 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml config.yaml +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json wandb-summary.json +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt requirements.txt +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log output.log +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/diff.patch diff.patch +2025-01-01 13:42:34,427 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:42:34,433 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 media/videos/eval/3_eval_3_252c2cdff8a70260e611.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 media/videos/eval/2_eval_2_798887158b66dda571d4.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 media/videos/eval/4_eval_4_031275cc1ef6ebf2ca3f.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 media/videos/eval/0_eval_0_568bf5b9d4210e7eaeb5.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 media/videos/eval/1_eval_1_286da0659dcf7249eb98.mp4 +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json media/table/eval/total_success_rate_ci_6_a19350b7b1a9e28e296d.table.json +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-01 13:42:34,435 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:42:34,435 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:34,436 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,436 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:42:34,436 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,436 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:42:34,436 INFO SenderThread:1805851 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:42:34,580 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:34,581 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:34,616 INFO wandb-upload_1:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/config.yaml +2025-01-01 13:42:34,620 INFO wandb-upload_2:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/wandb-summary.json +2025-01-01 13:42:34,686 INFO wandb-upload_0:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/requirements.txt +2025-01-01 13:42:34,698 INFO wandb-upload_3:1805851 [upload_job.py:push():130] Uploaded file /tmp/tmpnbyrd4cowandb/6avruo3v-events.out.tfevents.1735756539.leonmkim-ROG-Strix-G15CS-G15CS.1805676.0 +2025-01-01 13:42:34,699 INFO wandb-upload_4:1805851 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/files/output.log +2025-01-01 13:42:34,899 INFO Thread-11 (_thread_body):1805851 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:42:34,899 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,899 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:42:34,900 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,900 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:42:34,900 INFO SenderThread:1805851 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:42:34,901 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:42:34,901 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:34,901 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:42:34,902 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:34,902 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:42:34,902 INFO SenderThread:1805851 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:42:35,079 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:35,079 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:42:35,079 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:35,079 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:42:35,080 INFO SenderThread:1805851 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:42:35,080 DEBUG SenderThread:1805851 [sender.py:send():379] send: final +2025-01-01 13:42:35,080 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:42:35,080 DEBUG SenderThread:1805851 [sender.py:send():379] send: footer +2025-01-01 13:42:35,080 INFO HandlerThread:1805851 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:42:35,081 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: defer +2025-01-01 13:42:35,081 INFO SenderThread:1805851 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:42:35,081 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:35,081 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:35,082 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:42:35,082 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:42:35,083 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:42:35,086 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:42:35,086 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:42:35,087 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:42:35,139 DEBUG SenderThread:1805851 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:42:35,140 INFO MainThread:1805851 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:42:35,140 DEBUG HandlerThread:1805851 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:42:35,140 INFO HandlerThread:1805851 [handler.py:finish():882] shutting down handler +2025-01-01 13:42:36,082 INFO WriterThread:1805851 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb +2025-01-01 13:42:36,139 INFO SenderThread:1805851 [sender.py:finish():1615] shutting down sender +2025-01-01 13:42:36,140 INFO SenderThread:1805851 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:42:36,140 INFO SenderThread:1805851 [file_pusher.py:join():175] waiting for file pusher diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug.log b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d34b7f3abaf8f32719fd44ab574d2008ee277c86 --- /dev/null +++ b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:35:39,044 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Configure stats pid to 1805676 +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/settings +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug.log +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/logs/debug-internal.log +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx'} +2025-01-01 13:35:39,045 INFO MainThread:1805676 [wandb_init.py:init():619] starting backend +2025-01-01 13:35:39,046 INFO MainThread:1805676 [wandb_init.py:init():623] setting up manager +2025-01-01 13:35:39,052 INFO MainThread:1805676 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:35:39,053 INFO MainThread:1805676 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:35:39,067 INFO MainThread:1805676 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:35:39,073 INFO MainThread:1805676 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:35:39,305 INFO MainThread:1805676 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:35:39,372 INFO MainThread:1805676 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:35:39,372 INFO MainThread:1805676 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:35:39,709 INFO MainThread:1805676 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:35:39,710 INFO MainThread:1805676 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:35:39,711 INFO MainThread:1805676 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/kp7i5uxx/tb, True +2025-01-01 13:36:10,475 INFO MainThread:1805676 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 13:42:36,141 WARNING MsgRouterThr:1805676 [router.py:message_loop():77] message_loop has been closed diff --git a/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4224e9f6c64a2f6584eeb81dee85403bfd3f46bb Binary files /dev/null and b/kp7i5uxx/wandb/run-20250101_133539-kp7i5uxx/run-kp7i5uxx.wandb differ diff --git a/niwmozqn/episode_rosbags/episode_0_2025-01-01-13-52-01.bag b/niwmozqn/episode_rosbags/episode_0_2025-01-01-13-52-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..98e62bf53e72988ee4341f583f5c80643a0bb1ca --- /dev/null +++ b/niwmozqn/episode_rosbags/episode_0_2025-01-01-13-52-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b782567286de377e11c0e03058f86286006715e7acb4cb69c1bb6fe3d6fe58 +size 2605988071 diff --git a/niwmozqn/episode_rosbags/episode_1_2025-01-01-13-52-57.bag b/niwmozqn/episode_rosbags/episode_1_2025-01-01-13-52-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..3a59a53f324db3b8635afa36d68b59d6417573dc --- /dev/null +++ b/niwmozqn/episode_rosbags/episode_1_2025-01-01-13-52-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:739a6b9f0f7d3c97f135094b6a31c7b8473e69f61b54406baddee93b453d8708 +size 2605921608 diff --git a/niwmozqn/episode_rosbags/episode_2_2025-01-01-13-53-54.bag b/niwmozqn/episode_rosbags/episode_2_2025-01-01-13-53-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..ffc862bc8e892341c328fbcec22bb1be757c6e9a --- /dev/null +++ b/niwmozqn/episode_rosbags/episode_2_2025-01-01-13-53-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01c92b6a16e954ecb1ee927aafd4d550436e3c9d0f66d2cf44a79b771297e725 +size 2605933356 diff --git a/niwmozqn/episode_rosbags/episode_3_2025-01-01-13-54-52.bag b/niwmozqn/episode_rosbags/episode_3_2025-01-01-13-54-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..b24219b93b3cac39cab132f5e31dcbd5cca0a7c9 --- /dev/null +++ b/niwmozqn/episode_rosbags/episode_3_2025-01-01-13-54-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5853eb920beb3866420947e2f92661f0d182cf3cf9bfeeb6537e9f75740e6c0f +size 2603665934 diff --git a/niwmozqn/episode_rosbags/episode_4_2025-01-01-13-55-53.bag b/niwmozqn/episode_rosbags/episode_4_2025-01-01-13-55-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..aaf96d85d1660b569070eaa1babc3d2af4ec30b7 --- /dev/null +++ b/niwmozqn/episode_rosbags/episode_4_2025-01-01-13-55-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5f6cae9bb4fb9cc3cd9938c1fc455e468461c376b52c0b34ea0dd4ba2d6587 +size 2605180212 diff --git a/snapshot_12435.pt b/snapshot_12435.pt new file mode 100644 index 0000000000000000000000000000000000000000..2486ebd835e1a84fb7b187613f49d59358910d5a --- /dev/null +++ b/snapshot_12435.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d999a1ecff9560be438e321971a2e2c67b1f0ec5218a8f602b2f40104b08701 +size 910217934